# -*- coding: utf-8 -*-
from setuphelpers import *
from setupdevhelpers import *
import os
import sys
import bs4 as BeautifulSoup
from urllib.parse import urlparse, urljoin
# ext_lib_dir = os.sep.join([sys.executable.rsplit(os.path.sep, 1)[0].replace("/bin", "").replace("\\Scripts", ""), "libdev"])
# sys.path.insert(0, ext_lib_dir)
package_name = "pip"
python_version = "".join(str(x) for x in sys.version_info[:2])
def update_package():
# Declaring local variables
package_updated = False
proxies = get_proxies()
if not proxies:
proxies = get_proxies_from_wapt_console()
api_url = f"https://pypi.org/pypi/{package_name}/json"
if params.get("running_as_luti"):
index_url = "https://pip.ad.tranquil.it/"
else:
index_url = None
# fixed_version = "9.5.0"
fixed_version = None
# Downloading whl files in package_name folder
if isfile("requirements.txt"):
pip_download_cmd = f'"{get_waptpython_path()}" -m pip download -r requirements.txt --only-binary=:all: --dest "{package_name}"'
else:
pip_download_cmd = f'"{get_waptpython_path()}" -m pip download {package_name} --only-binary=:all: --dest "{package_name}"'
if index_url:
pip_download_cmd += f' --index-url="{index_url}" --trusted-host="{index_url.split("//")[1].split("/")[0]}"'
pip_download_cmd += f' --proxy=""'
elif proxies:
pip_download_cmd += f' --proxy="{proxies["http"]}"'
print(run(pip_download_cmd))
# Getting latest version information from official sources
print("API used is: %s" % api_url)
json_load = wgets(api_url, proxies=proxies, as_json=True)
if fixed_version:
json_search = json_load["releases"][fixed_version]
version = fixed_version
else:
json_search = json_load["urls"]
version = json_load["info"]["version"]
if not index_url:
for to_download in json_search:
filename = to_download["filename"]
if ".whl" in filename and ("-py3-" in filename or f"cp{python_version}" in filename):
download_url = to_download["url"]
latest_bin = package_name + os.sep + filename
if not isfile(latest_bin):
print("Downloading: %s" % latest_bin)
wget(download_url, latest_bin, proxies=proxies)
else:
print("Binary is present: %s" % latest_bin)
else:
binaries_dict = bs_index_of_to_dict(index_url + package_name, proxies=proxies)
for to_download in binaries_dict.keys():
filename = to_download
if ".whl" in filename and ("-py3-" in filename or f"cp{python_version}" in filename) and version in filename:
download_url = binaries_dict[to_download]
latest_bin = package_name + os.sep + filename
if not isfile(latest_bin):
print("Downloading: %s" % latest_bin)
wget(download_url, latest_bin, proxies=proxies)
else:
print("Binary is present: %s" % latest_bin)
# Deleting outdated binaries
for f in glob.glob("*.whl"):
print("Removing: %s" % f)
remove_file(f)
# Deleting other binaries
for f in glob.glob(f'{package_name}/{json_load["info"]["name"]}*.whl'): # filename.split("-")[0]
if not version in f:
print("Removing: %s" % f)
remove_file(f)
# Deleting other dirs
for dir_name in glob.glob("*/"):
dir_name = dir_name.split(os.sep)[0]
if not dir_name in ["WAPT", "__pycache__"] and not package_name in dir_name:
remove_tree(dir_name)
# generate_requirements_txt
if not isfile("requirements.txt"):
generate_requirements_txt(package_name, glob.glob(f"{package_name}/*.whl"))
else:
with open("requirements.txt") as f:
requirements_list = f.read().split("\n")
requirements_list = [a.split("==")[0] for a in requirements_list]
for package in glob.glob(f"{package_name}/*.whl"):
package_info = package.split(os.sep)[-1].split("-")
name = package_info[0].replace("_", "-").lower()
if not name in requirements_list:
remove_file(package)
# control autocompletion
api_url = f"https://pypi.org/pypi/{package_name}/json"
json_load = wgets(api_url, proxies=proxies, as_json=True)
control.package = control.package.split("-", 1)[0] + "-waptpython-" + json_load["info"]["name"].lower()
control.name = "waptpython " + json_load["info"]["name"]
control.installed_size = get_size(package_name) * 3
if json_load["info"].get("keywords") is not None and json_load["info"]["keywords"]:
print(f'You may wanna add: "{json_load["info"]["keywords"].lower()}" in: "control.keywords"')
control.description = json_load["info"]["summary"]
if json_load["info"]["project_urls"].get("Changelog") is not None and json_load["info"]["project_urls"]["Changelog"]:
control.changelog = json_load["info"]["project_urls"]["Changelog"]
control.sources = json_load["info"]["package_url"] + "#files"
if json_load["info"]["project_urls"].get("Homepage") is not None and json_load["info"]["project_urls"]["Homepage"]:
control.homepage = json_load["info"]["project_urls"]["Homepage"]
# Changing version of the package
if Version(version, 4) > Version(control.get_software_version(), 4):
print("Software version updated (from: %s to: %s)" % (control.get_software_version(), Version(version)))
package_updated = True
else:
print("Software version up-to-date (%s)" % Version(version))
control.set_software_version(version)
control.save_control_to_wapt()
# Validating update-package-sources
return package_updated
# # Changing version of the package and validating update-package-sources
# return complete_control_version(control, version)
def bs_index_of_to_dict(url, **kwargs):
r"""Parse html "Index of" web page with BeautifulSoup and get a dict of the result
Args:
url (str): url of the web page to parse
Return:
result: a dict of the index URLs
"""
result = {}
for elem in bs_find_all(url, "a", "href"):
if not "://" in elem["href"]:
# absolute_link = urljoin(url, url + "/" + elem["href"] if not url.endswith("/") else url + elem["href"])
absolute_link = url + "/" + elem["href"] if not url.endswith("/") else url + elem["href"]
else:
absolute_link = elem["href"]
key = elem.text
if key.endswith("..>"): # if key.endswith("..>"):
key = elem["href"]
result[key] = absolute_link
return result
def is_url(x):
try:
result = urlparse(x)
return all([result.scheme, result.netloc])
except:
return False
def bs_find_all(url, element, attribute=None, value=None, user_agent=None, proxies=None, features="html.parser", **kwargs):
"""
Parse an HTML or XML web page with BeautifulSoup and retrieve a list of all matching results.
Args:
url (str): URL of the web page or string to parse.
element (str): Searched element.
attribute (str): Selected attribute of the element.
value (str): Value of the selected attribute.
user_agent (str): Specify a user-agent if needed.
proxies (dict): Specify proxies if needed.
features (str): BeautifulSoup feature to use.
**kwargs: Additional parameters for the requests library.
Returns:
list: List of bs4.element.Tag objects representing the matching elements.
Examples:
>>> bs_find_all('https://www.w3.org/', 'a', 'title', 'Open Web Platform testing')[0]['href']
'https://web-platform-tests.org/'
>>> bs_find_all('https://www.w3.org/', 'span', 'class', 'alt-logo')[0].string
'W3C'
.. versionadded:: 2.0
.. versionchanged:: 2.5
Function can now parse string content of a page or reparse a "bs_result". It is now possible to parse a specific attribute.
"""
url = str(url)
if is_url(url):
if user_agent:
page = requests.get(url, proxies=proxies, headers={"User-Agent": user_agent}, **kwargs).text
else:
page = requests.get(url, proxies=proxies, **kwargs).text
else:
page = url
soup = BeautifulSoup.BeautifulSoup(page, features=features)
if value:
return soup.find_all(element, {attribute: value})
elif attribute:
return soup.find_all(element, attrs={attribute: True})
else:
return soup.find_all(element)
def generate_requirements_txt(package_name, packages_list):
latest_versions = {}
for package in packages_list:
package_info = package.split(os.sep)[-1].split("-")
name = package_info[0].replace("_", "-").lower()
version = package_info[1]
if name not in latest_versions:
latest_versions[name] = version
else:
if version > latest_versions[name]:
latest_versions[name] = version
# if name == package_name:
# requirements_content = [f"{name}=={version}" for name, version in latest_versions.items()]
# else:
# requirements_content = [f"{name}" for name, version in latest_versions.items()]
requirements_content = [f"{name}" for name, version in latest_versions.items()]
if requirements_content:
with open("requirements.txt", "w", newline="\n") as requirements_file:
requirements_file.write("\n".join(requirements_content))
def get_waptpython_path():
if get_os_name() == "Windows":
return makepath(WAPT.wapt_base_dir, "waptpython.exe")
else:
return makepath(WAPT.wapt_base_dir, "waptpython.sh")
def get_size(start_path="."):
total_size = 0
for dirpath, dirnames, filenames in os.walk(start_path):
for f in filenames:
fp = os.path.join(dirpath, f)
# skip if it is symbolic link
if not os.path.islink(fp):
total_size += os.path.getsize(fp)
return total_size
