commit 8370db2d59690b60acc889feab32854ed2cd4078 Author: topilski Date: Wed Dec 18 03:21:57 2019 -0500 Init sources diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..259e3e1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,92 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +# fastogt tmp files +.idea/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..65c5ca8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/README.md b/README.md new file mode 100644 index 0000000..9f2f182 --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +About PyFastoGT +=============== + +FastoGT python files. + +Dependencies +======== +`setuptools` + +Install +======== +`python3 setup.py install` diff --git a/pyfastogt/__init__.py b/pyfastogt/__init__.py new file mode 100644 index 0000000..d3f5a12 --- /dev/null +++ b/pyfastogt/__init__.py @@ -0,0 +1 @@ + diff --git a/pyfastogt/__version__.py b/pyfastogt/__version__.py new file mode 100644 index 0000000..d11b18e --- /dev/null +++ b/pyfastogt/__version__.py @@ -0,0 +1,3 @@ +VERSION = (1, 0, 0) + +__version__ = '.'.join(map(str, VERSION)) diff --git a/pyfastogt/build_utils.py b/pyfastogt/build_utils.py new file mode 100644 index 0000000..5ac2f44 --- /dev/null +++ b/pyfastogt/build_utils.py @@ -0,0 +1,388 @@ +import os +import stat +import shutil +import subprocess +from pyfastogt import system_info, utils + + +class BuildSystem: + def __init__(self, name: str, cmd_line: list, cmake_generator_arg: str): + self.name_ = name + self.cmd_line_ = cmd_line + self.cmake_generator_arg_ = cmake_generator_arg + + def cmake_generator_arg(self) -> str: + return self.cmake_generator_arg_ + + def name(self) -> str: + return self.name_ + + def cmd_line(self) -> list: # cmd + args + return self.cmd_line_ + + +SUPPORTED_BUILD_SYSTEMS = [BuildSystem('ninja', ['ninja'], 'Ninja'), + BuildSystem('single_make', ['make'], 'Unix Makefiles'), + BuildSystem('make', ['make', '-j2'], 'Unix Makefiles'), + BuildSystem('gmake', ['gmake', '-j2'], 'Unix Makefiles')] + + +def get_supported_build_system_by_name(name) -> BuildSystem: + return next((x for x in SUPPORTED_BUILD_SYSTEMS if x.name() == name), None) + + +class BuildError(Exception): + def __init__(self, value): + self.value_ = value + + def __str__(self): + return self.value_ + + +# must be in cmake folder +def build_command_cmake(prefix_path: str, cmake_flags: list, build_type='RELEASE', cmake_project_root_abs_path='..', + build_system=get_supported_build_system_by_name('ninja')): + if not os.path.exists(cmake_project_root_abs_path): + raise BuildError('invalid cmake_project_root_path: %s' % cmake_project_root_abs_path) + + abs_prefix_path = os.path.expanduser(prefix_path) + cmake_line = ['cmake', cmake_project_root_abs_path, '-G', build_system.cmake_generator_arg(), + '-DCMAKE_BUILD_TYPE=%s' % build_type] + cmake_line.extend(cmake_flags) + cmake_line.extend(['-DCMAKE_INSTALL_PREFIX=%s' % abs_prefix_path]) + try: + build_dir_name = 'build_cmake_%s' % build_type.lower() + if os.path.exists(build_dir_name): + shutil.rmtree(build_dir_name) + + os.mkdir(build_dir_name) + os.chdir(build_dir_name) + subprocess.call(cmake_line) + make_line = build_system.cmd_line() + subprocess.call(make_line) + make_line.append('install') + subprocess.call(make_line) + if hasattr(shutil, 'which') and shutil.which('ldconfig'): + subprocess.call(['ldconfig']) + except Exception as ex: + ex_str = str(ex) + raise BuildError(ex_str) + + +# must be in configure folder +def build_command_configure(compiler_flags: list, prefix_path, executable='./configure', + build_system=get_supported_build_system_by_name('make')): + # +x for exec file + st = os.stat(executable) + os.chmod(executable, st.st_mode | stat.S_IEXEC) + + abs_prefix_path = os.path.expanduser(prefix_path) + compile_cmd = [executable, '--prefix={0}'.format(abs_prefix_path)] + compile_cmd.extend(compiler_flags) + subprocess.call(compile_cmd) + make_line = build_system.cmd_line() + subprocess.call(make_line) + make_line.append('install') + subprocess.call(make_line) + if hasattr(shutil, 'which') and shutil.which('ldconfig'): + subprocess.call(['ldconfig']) + + +def generate_fastogt_git_path(repo_name) -> str: + return 'https://github.com/fastogt/%s' % repo_name + + +class BuildRequest(object): + OPENSSL_SRC_ROOT = "https://www.openssl.org/source/" + ARCH_OPENSSL_COMP = "gz" + ARCH_OPENSSL_EXT = "tar." + ARCH_OPENSSL_COMP + + CMAKE_SRC_ROOT = "https://github.com/Kitware/CMake/releases/download" + ARCH_CMAKE_COMP = "gz" + ARCH_CMAKE_EXT = "tar." + ARCH_CMAKE_COMP + + MESON_SRC_ROOT = "https://github.com/mesonbuild/meson/releases/download" + MESON_ARCH_COMP = "gz" + MESON_ARCH_EXT = "tar." + MESON_ARCH_COMP + + def __init__(self, platform: str, arch_name: str, dir_path: str, prefix_path: str): + platform_or_none = system_info.get_supported_platform_by_name(platform) + if not platform_or_none: + raise BuildError('invalid platform') + + arch_or_none = platform_or_none.get_architecture_by_arch_name(arch_name) + if not arch_or_none: + raise BuildError('invalid arch') + + if not prefix_path: + prefix_path = arch_or_none.default_install_prefix_path() + abs_prefix_path = os.path.expanduser(prefix_path) + + packages_types = platform_or_none.package_types() + build_platform = platform_or_none.make_platform_by_arch(arch_or_none, packages_types) + + env_pkg_path = os.environ.get('PKG_CONFIG_PATH') + add_env_pkg_path = '%s/lib/pkgconfig/' % abs_prefix_path + os.environ['PKG_CONFIG_PATH'] = '{0}:{1}'.format(env_pkg_path, + add_env_pkg_path) if env_pkg_path else add_env_pkg_path + env_ld_library_path = os.environ.get('LD_LIBRARY_PATH') + add_env_ld_library_path = '%s/lib' % abs_prefix_path + os.environ['LD_LIBRARY_PATH'] = '{0}:{1}'.format(env_ld_library_path, + add_env_ld_library_path) if env_ld_library_path else add_env_ld_library_path + env_path = os.environ.get('PATH') + add_env_path = '%s/bin' % abs_prefix_path + os.environ['PATH'] = '{0}:{1}'.format(env_path, add_env_path) if env_path else add_env_path + env = build_platform.env_variables() + for key, value in env.items(): + os.environ[key] = value + + self.platform_ = build_platform + build_dir_path = os.path.abspath(dir_path) + if os.path.exists(build_dir_path): + shutil.rmtree(build_dir_path) + + os.mkdir(build_dir_path) + os.chdir(build_dir_path) + + self.build_dir_path_ = build_dir_path + self.prefix_path_ = abs_prefix_path + print("Build request for platform: {0}({1}) created".format(build_platform.name(), arch_or_none.name())) + + def platform(self): + return self.platform_ + + def platform_name(self) -> str: + return self.platform_.name() + + def build_dir_path(self): + return self.build_dir_path_ + + def prefix_path(self): + return self.prefix_path_ + + def build_snappy(self): + self._clone_and_build_via_cmake(generate_fastogt_git_path('snappy'), + ['-DBUILD_SHARED_LIBS=OFF', '-DSNAPPY_BUILD_TESTS=OFF']) + + def build_jsonc(self): + self._clone_and_build_via_cmake(generate_fastogt_git_path('json-c'), ['-DBUILD_SHARED_LIBS=OFF']) + + def build_libev(self): + libev_compiler_flags = ['--with-pic', '--disable-shared', '--enable-static'] + self._clone_and_build_via_autogen(generate_fastogt_git_path('libev'), libev_compiler_flags) + + def build_cpuid(self): + cpuid_compiler_flags = ['--disable-shared', '--enable-static'] + + pwd = os.getcwd() + cloned_dir = utils.git_clone(generate_fastogt_git_path('libcpuid')) + os.chdir(cloned_dir) + + platform_name = self.platform_name() + if platform_name == 'macosx': + libtoolize_cpuid = ['glibtoolize'] + else: + libtoolize_cpuid = ['libtoolize'] + subprocess.call(libtoolize_cpuid) + + autoreconf_cpuid = ['autoreconf', '--install'] + subprocess.call(autoreconf_cpuid) + + self._build_via_configure(cpuid_compiler_flags) + + os.chdir(pwd) + + def update_pyfastogt(self): + self._clone_and_build_via_python3(generate_fastogt_git_path('pyfastogt')) + + def build_common(self, with_qt=False): + cmake_flags = [] + if with_qt: + cmake_flags.append('-DQT_ENABLED=ON') + + self._clone_and_build_via_cmake(generate_fastogt_git_path('common'), cmake_flags) + + def build_fastotv_protocol(self): + cmake_flags = [] + self._clone_and_build_via_cmake(generate_fastogt_git_path('fastotv_protocol'), cmake_flags) + + def build_fastoplayer(self): + cmake_flags = [] + self._clone_and_build_via_cmake(generate_fastogt_git_path('fastoplayer'), cmake_flags) + + def build_cmake(self, version): + compiler_flags = [] + url = '{0}/v{1}/cmake-{1}.{2}'.format(self.CMAKE_SRC_ROOT, version, self.ARCH_CMAKE_EXT) + self._download_and_build_via_configure(url, compiler_flags) + + def build_meson(self, version): + url = '{0}/{1}/meson-{1}.{2}'.format(self.MESON_SRC_ROOT, version, self.MESON_ARCH_EXT) + self._download_and_build_via_python3(url) + + def build_openssl(self, version, have_shared=False): + compiler_flags = ['no-tests'] + if not have_shared: + compiler_flags.append('no-shared') + + platform = self.platform() + if platform.name() == 'android': + compiler_flags.append('no-asm') + + # compiler_flags.append('--openssldir={0}'.format(self.prefix_path_)) + compiler_flags.append('--libdir=lib') + + url = '{0}openssl-{1}.{2}'.format(self.OPENSSL_SRC_ROOT, version, self.ARCH_OPENSSL_EXT) + # download + pwd = os.getcwd() + file_path = utils.download_file(url) + extracted_folder = utils.extract_file(file_path) + os.chdir(extracted_folder) + build_command_configure(compiler_flags, self.prefix_path_, './config', + get_supported_build_system_by_name('single_make')) + os.chdir(pwd) + + # install packages + def _install_package(self, name: str): + self.platform_.install_package(name) + + def _install_via_python3(self, name: str): + python3_line = ['pip3', 'install', name] + subprocess.call(python3_line) + + # clone + def _clone_and_build_via_cmake(self, url: str, cmake_flags: list, branch=None, remove_dot_git=True): + pwd = os.getcwd() + cloned_dir = utils.git_clone(url, branch, remove_dot_git) + os.chdir(cloned_dir) + self._build_via_cmake(cmake_flags) + os.chdir(pwd) + + def _clone_and_build_via_meson(self, url: str, meson_flags: list, branch=None, remove_dot_git=True): + pwd = os.getcwd() + cloned_dir = utils.git_clone(url, branch, remove_dot_git) + os.chdir(cloned_dir) + self._build_via_meson(meson_flags) + os.chdir(pwd) + + def _clone_and_build_via_configure(self, url: str, compiler_flags: list, executable='./configure', + use_platform_flags=True, branch=None, remove_dot_git=True): + pwd = os.getcwd() + cloned_dir = utils.git_clone(url, branch, remove_dot_git) + os.chdir(cloned_dir) + self._build_via_configure(compiler_flags, executable, use_platform_flags) + os.chdir(pwd) + + def _clone_and_build_via_autogen(self, url: str, compiler_flags: list, executable='./configure', + use_platform_flags=True, branch=None, + remove_dot_git=True): + pwd = os.getcwd() + cloned_dir = utils.git_clone(url, branch, remove_dot_git) + os.chdir(cloned_dir) + self._build_via_autogen(compiler_flags, executable, use_platform_flags) + os.chdir(pwd) + + def _clone_and_build_via_python3(self, url: str, branch=None, + remove_dot_git=True): + pwd = os.getcwd() + cloned_dir = utils.git_clone(url, branch, remove_dot_git) + os.chdir(cloned_dir) + python3_line = ['python3', 'setup.py', 'install'] + subprocess.call(python3_line) + os.chdir(pwd) + + # download + def _download_and_build_via_cmake(self, url: str, cmake_flags: list): + pwd = os.getcwd() + file_path = utils.download_file(url) + extracted_folder = utils.extract_file(file_path) + os.chdir(extracted_folder) + self._build_via_cmake(cmake_flags) + os.chdir(pwd) + + def _download_and_build_via_bootstrap(self, url: str, compiler_flags: list, executable='./configure', + use_platform_flags=True): + pwd = os.getcwd() + file_path = utils.download_file(url) + extracted_folder = utils.extract_file(file_path) + os.chdir(extracted_folder) + self._build_via_bootstrap(compiler_flags, executable, use_platform_flags) + os.chdir(pwd) + + def _download_and_build_via_autogen(self, url: str, compiler_flags: list, executable='./configure', + use_platform_flags=True): + pwd = os.getcwd() + file_path = utils.download_file(url) + extracted_folder = utils.extract_file(file_path) + os.chdir(extracted_folder) + self._build_via_autogen(compiler_flags, executable, use_platform_flags) + os.chdir(pwd) + + def _download_and_build_via_python3(self, url: str): + pwd = os.getcwd() + file_path = utils.download_file(url) + extracted_folder = utils.extract_file(file_path) + os.chdir(extracted_folder) + python3_line = ['python3', 'setup.py', 'install'] + subprocess.call(python3_line) + os.chdir(pwd) + + def _download_and_build_via_meson(self, url: str, compiler_flags: list, + build_system=get_supported_build_system_by_name('ninja')): + pwd = os.getcwd() + file_path = utils.download_file(url) + extracted_folder = utils.extract_file(file_path) + os.chdir(extracted_folder) + self._build_via_meson(compiler_flags, build_system) + os.chdir(pwd) + + def _download_and_build_via_configure(self, url: str, compiler_flags: list, executable='./configure', + use_platform_flags=True): + pwd = os.getcwd() + file_path = utils.download_file(url) + extracted_folder = utils.extract_file(file_path) + os.chdir(extracted_folder) + self._build_via_configure(compiler_flags, executable, use_platform_flags) + os.chdir(pwd) + + # build + def _build_via_autogen(self, compiler_flags: list, executable='./configure', use_platform_flags=True): + autogen_line = ['sh', 'autogen.sh'] + subprocess.call(autogen_line) + self._build_via_configure(compiler_flags, executable, use_platform_flags) + + def _build_via_bootstrap(self, compiler_flags: list, executable='./configure', use_platform_flags=True): + autogen_line = ['sh', 'bootstrap'] + subprocess.call(autogen_line) + self._build_via_configure(compiler_flags, executable, use_platform_flags) + + def _build_via_meson(self, compiler_flags: list, build_system=get_supported_build_system_by_name('ninja')): + build_dir_name = 'build_meson' + os.mkdir(build_dir_name) + os.chdir(build_dir_name) + abs_prefix_path = os.path.expanduser(self.prefix_path_) + meson_line = ['meson', '--prefix', abs_prefix_path, '--libdir', abs_prefix_path + '/lib'] + meson_line.extend(compiler_flags) + subprocess.call(meson_line) + make_line = build_system.cmd_line() + subprocess.call(make_line) + make_line.append('install') + subprocess.call(make_line) + + # raw build + def _build_via_cmake(self, cmake_flags: list, build_type='RELEASE', use_platform_flags=True): + cmake_flags_extended = cmake_flags + if use_platform_flags: + cmake_flags_extended.extend(self.platform_.cmake_specific_flags()) + build_command_cmake(self.prefix_path_, cmake_flags, build_type) + + def _build_via_cmake_double(self, cmake_flags: list, build_type='RELEASE', use_platform_flags=True): + cmake_flags_extended = cmake_flags + if use_platform_flags: + cmake_flags_extended.extend(self.platform_.cmake_specific_flags()) + build_command_cmake(self.prefix_path_, cmake_flags, build_type, '../../..') + + def _build_via_configure(self, compiler_flags: list, executable='./configure', use_platform_flags=True): + compiler_flags_extended = compiler_flags + if use_platform_flags: + compiler_flags_extended.extend(self.platform_.configure_specific_flags()) + build_command_configure(compiler_flags_extended, self.prefix_path_, executable) diff --git a/pyfastogt/run_command.py b/pyfastogt/run_command.py new file mode 100644 index 0000000..e3e35b7 --- /dev/null +++ b/pyfastogt/run_command.py @@ -0,0 +1,127 @@ +import re +import subprocess + + +class MessageType: + STATUS = 1 + MESSAGE = 2 + + +class Message(object): + def __init__(self, message, type): + self.message_ = message + self.type_ = type + + def message(self): + return self.message_ + + def type(self): + return self.type_ + + +class Policy(object): + def __init__(self, cb=None): + self.progress_ = 0.0 + self.cb_ = cb + + def process(self, message): + if self.cb_: + self.cb_(self.progress_, message) + + def update_progress_message(self, progress, message): + self.progress_ = progress + msg = Message(message, MessageType.STATUS) + self.process(msg) + + +class CommonPolicy(Policy): + def __init__(self, cb): + Policy.__init__(self, cb) + + +class CmakePolicy(Policy): + def __init__(self, cb): + Policy.__init__(self, cb) + + def process(self, message): + self.progress_ += 1.0 + super(CmakePolicy, self).process(message) + + def update_progress_message(self, progress, message): + super(CmakePolicy, self).update_progress_message(progress, message) + + +class MakePolicy(Policy): + def __init__(self, cb): + Policy.__init__(self, cb) + + def process(self, message): + if message.type() != MessageType.MESSAGE: + super(MakePolicy, self).process(message) + return + + cur = self.parse_message_to_get_percent(message.message()) + if not cur: + return + + self.progress_ = cur + super(MakePolicy, self).process(message) + + def update_progress_message(self, progress, message): + super(MakePolicy, self).update_progress_message(progress, message) + + def parse_message_to_get_percent(self, message): + if not message: + return None + + res = re.search(r'\A\[ (\d+)%\]', message) + if res: + return float(res.group(1)) + + return None + + +class NinjaPolicy(Policy): + def __init__(self, cb): + Policy.__init__(self, cb) + + def process(self, message): + if message.type() != MessageType.MESSAGE: + super(NinjaPolicy, self).process(message) + return + + cur, total = self.parse_message_to_get_range(message.message()) + if not cur and not total: + return + + self.progress_ = cur / total * 100.0 + super(NinjaPolicy, self).process(message) + + def update_progress_message(self, progress, message): + super(NinjaPolicy, self).update_progress_message(progress, message) + + def parse_message_to_get_range(self, message): + if not message: + return None, None + + res = re.search(r'\A\[(\d+)/(\d+)\]', message) + if res: + return float(res.group(1)), float(res.group(2)) + + return None, None + + +def run_command_cb(cmd: list, policy=Policy()): + try: + policy.update_progress_message(0.0, 'Command {0} started'.format(cmd)) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) + for output in process.stdout: + line = output.strip() + policy.process(Message(line.decode("utf-8"), MessageType.MESSAGE)) + rc = process.poll() + policy.update_progress_message(100.0, 'Command {0} finished successfully'.format(cmd)) + except subprocess.CalledProcessError as ex: + policy.update_progress_message(100.0, 'Command {0} finished with exception {1}'.format(cmd, str(ex))) + raise ex + + return rc diff --git a/pyfastogt/system_info.py b/pyfastogt/system_info.py new file mode 100644 index 0000000..55bc44a --- /dev/null +++ b/pyfastogt/system_info.py @@ -0,0 +1,303 @@ +import platform +import distro +import subprocess +import os +from abc import ABCMeta, abstractmethod + + +class Architecture(object): + def __init__(self, arch: str, bit: int, default_install_prefix_path: str): + self.name_ = arch + self.bit_ = bit + self.default_install_prefix_path_ = default_install_prefix_path + + def name(self) -> str: + return self.name_ + + def bit(self) -> int: + return self.bit_ + + def default_install_prefix_path(self) -> str: + return self.default_install_prefix_path_ + + +class Platform(metaclass=ABCMeta): + def __init__(self, name: str, architecture: Architecture, package_types: list): + self.name_ = name + self.architecture_ = architecture + self.package_types_ = package_types + + def name(self) -> str: + return self.name_ + + def architecture(self) -> Architecture: + return self.architecture_ + + def package_types(self) -> list: + return self.package_types_ + + @abstractmethod + def install_package(self, name: str): + pass + + def env_variables(self) -> dict: + return {} + + def cmake_specific_flags(self) -> list: + return [] + + def configure_specific_flags(self) -> list: + return [] + + +class SupportedPlatforms(metaclass=ABCMeta): + def __init__(self, name: str, architectures: [Architecture], package_types: list): + self.name_ = name + self.architectures_ = architectures + self.package_types_ = package_types + + def name(self) -> str: + return self.name_ + + def architectures(self) -> [Architecture]: + return self.architectures_ + + def package_types(self) -> list: + return self.package_types_ + + def get_architecture_by_arch_name(self, name: str) -> Architecture: + return next((x for x in self.architectures_ if x.name() == name), None) + + @abstractmethod + def make_platform_by_arch(self, arch: Architecture, package_types: list) -> Platform: # factory method + pass + + +def linux_get_dist(): + """ + Return the running distribution group + RHEL: RHEL, CENTOS, FEDORA + DEBIAN: UBUNTU, DEBIAN, LINUXMINT + """ + linux_tuple = distro.linux_distribution() + dist_name = linux_tuple[0] + dist_name_upper = dist_name.upper() + + if dist_name_upper.startswith(("RHEL", "CENTOS LINUX", "FEDORA", "AMAZON LINUX")): + return "RHEL" + elif dist_name_upper.startswith(("DEBIAN", "UBUNTU", "LINUXMINT", "RASPBIAN GNU/LINUX")): + return "DEBIAN" + elif dist_name_upper.startswith(("ARCH")): + return "ARCH" + raise NotImplemented("Unknown platform '%s'" % dist_name) + + +# Linux platforms + +class DebianPlatform(Platform): + def __init__(self, arch: Architecture, package_types: list): + Platform.__init__(self, 'linux', arch, package_types) + + def install_package(self, name: str): + subprocess.call(['apt-get', '-y', '--no-install-recommends', 'install', name]) + + +class RedHatPlatform(Platform): + def __init__(self, arch: Architecture, package_types: list): + Platform.__init__(self, 'linux', arch, package_types) + + def install_package(self, name: str): + subprocess.call(['yum', '-y', 'install', name]) + + +class ArchPlatform(Platform): + def __init__(self, arch: Architecture, package_types: list): + Platform.__init__(self, 'linux', arch, package_types) + + def install_package(self, name: str): + subprocess.call(['pacman', '-S', '--noconfirm', name]) + + +class LinuxPlatforms(SupportedPlatforms): + def __init__(self): + SupportedPlatforms.__init__(self, 'linux', [Architecture('x86_64', 64, '/usr/local'), + Architecture('i386', 32, '/usr/local'), + Architecture('i686', 32, '/usr/local'), + Architecture('aarch64', 64, '/usr/local'), + Architecture('armv7l', 32, '/usr/local'), + Architecture('armv6l', 32, '/usr/local')], + ['DEB', 'RPM', 'TGZ']) + + def make_platform_by_arch(self, arch: Architecture, package_types: list) -> Platform: + distr = linux_get_dist() + if distr == 'DEBIAN': + return DebianPlatform(arch, package_types) + elif distr == 'RHEL': + return RedHatPlatform(arch, package_types) + elif distr == 'ARCH': + return ArchPlatform(arch, package_types) + raise NotImplemented("Unknown distribution '%s'" % distr) + + +# Windows platforms +class WindowsMingwPlatform(Platform): + def __init__(self, arch: Architecture, package_types: list): + Platform.__init__(self, 'windows', arch, package_types) + + def install_package(self, name: str): + subprocess.call(['pacman', '-S', '--noconfirm', name]) + + +class WindowsPlatforms(SupportedPlatforms): + def __init__(self): + SupportedPlatforms.__init__(self, 'windows', + [Architecture('x86_64', 64, '/mingw64'), + Architecture('AMD64', 64, '/mingw64'), + Architecture('i386', 32, '/mingw32'), + Architecture('i686', 32, '/mingw32')], + ['NSIS', 'ZIP']) + + def make_platform_by_arch(self, arch: Architecture, package_types: list) -> Platform: + return WindowsMingwPlatform(arch, package_types) + + +# MacOSX platforms +class MacOSXCommonPlatform(Platform): + def __init__(self, arch: Architecture, package_types: list): + Platform.__init__(self, 'macosx', arch, package_types) + + def install_package(self, name: str): + subprocess.call(['port', '-N', 'install', name]) + + +class MacOSXPlatforms(SupportedPlatforms): + def __init__(self): + SupportedPlatforms.__init__(self, 'macosx', [Architecture('x86_64', 64, '/usr/local')], ['DragNDrop', 'ZIP']) + + def make_platform_by_arch(self, arch: Architecture, package_types: list) -> Platform: + return MacOSXCommonPlatform(arch, package_types) + + +# FreeBSD platforms +class FreeBSDCommonPlatform(Platform): + def __init__(self, arch: Architecture, package_types: list): + Platform.__init__(self, 'freebsd', arch, package_types) + + def install_package(self, name: str): + subprocess.call(['pkg', 'install', '-y', name]) + + +class FreeBSDPlatforms(SupportedPlatforms): + def __init__(self): + SupportedPlatforms.__init__(self, 'freebsd', [Architecture('x86_64', 64, '/usr/local'), + Architecture('amd64', 64, '/usr/local')], ['TGZ']) + + def make_platform_by_arch(self, arch: Architecture, package_types: list) -> Platform: + return FreeBSDCommonPlatform(arch, package_types) + + +# Android platforms +ANDROID_PLATFORM_NUMBER = 16 +ANDROID_PLATFORM = 'android-%s' % ANDROID_PLATFORM_NUMBER +ANDROID_NDK = '~/Android/Sdk/ndk-bundle' + + +class AndroidCommonPlatform(Platform): + def __init__(self, arch: Architecture, package_types: list): + Platform.__init__(self, 'android', arch, package_types) + + def install_package(self, name: str): + raise NotImplementedError('You need to define a install_package method!') + + def env_variables(self) -> dict: + arch = self.architecture() + abs_prefix_path = os.path.expanduser(ANDROID_NDK) + return { + 'CC': '{0}/toolchains/llvm/prebuilt/linux-x86_64/bin/{1}-linux-androideabi{2}-clang'.format(abs_prefix_path, + arch.name(), + ANDROID_PLATFORM_NUMBER), + 'CXX': '{0}/toolchains/llvm/prebuilt/linux-x86_64/bin/{1}-linux-androideabi{2}-clang++'.format( + abs_prefix_path, arch.name(), ANDROID_PLATFORM_NUMBER)} + + def cmake_specific_flags(self) -> list: + abs_prefix_path = os.path.expanduser(ANDROID_NDK) + return ['-DCMAKE_TOOLCHAIN_FILE=%s/build/cmake/android.toolchain.cmake' % abs_prefix_path, + '-DANDROID_PLATFORM=%s' % ANDROID_PLATFORM] + + def configure_specific_flags(self) -> list: + arch = self.architecture() + return ['--host=%s-linux-androideabi' % arch.name()] + + +class AndroidPlatforms(SupportedPlatforms): + def __init__(self): + SupportedPlatforms.__init__(self, 'android', + [Architecture('armv7a', 32, + ANDROID_NDK + '/platforms/' + ANDROID_PLATFORM + '/arch-arm/usr/'), + Architecture('i686', 32, + ANDROID_NDK + '/platforms/' + ANDROID_PLATFORM + '/arch-x86/usr/'), + Architecture('x86_64', 64, + ANDROID_NDK + '/platforms/' + ANDROID_PLATFORM + '/arch-x86/usr/'), + Architecture('aarch64', 64, + ANDROID_NDK + '/platforms/' + ANDROID_PLATFORM + '/arch-x86/usr/')], + ['APK']) + + def make_platform_by_arch(self, arch: Architecture, package_types: list) -> Platform: + return AndroidCommonPlatform(arch, package_types) + + +SUPPORTED_PLATFORMS = [LinuxPlatforms(), WindowsPlatforms(), MacOSXPlatforms(), FreeBSDPlatforms(), AndroidPlatforms()] + + +def get_extension_by_package(package_type) -> str: + if package_type == 'DEB': + return 'deb' + elif package_type == 'RPM': + return 'rpm' + elif package_type == 'TGZ': + return 'tar.gz' + elif package_type == 'NSIS': + return 'exe' + elif package_type == 'ZIP': + return 'zip' + elif package_type == 'DragNDrop': + return 'dmg' + elif package_type == 'APK': + return 'apk' + else: + return 'unknown' + + +def get_os() -> str: + uname_str = platform.system() + if 'MINGW' in uname_str: + return 'windows' + elif 'MSYS' in uname_str: + return 'windows' + elif uname_str == 'Windows': + return 'windows' + elif uname_str == 'Linux': + return 'linux' + elif uname_str == 'Darwin': + return 'macosx' + elif uname_str == 'FreeBSD': + return 'freebsd' + elif uname_str == 'Android': + return 'android' + else: + return 'unknown' + + +def get_arch_name() -> str: + return platform.machine() + + +def get_supported_platform_by_name(name: str) -> SupportedPlatforms: + return next((x for x in SUPPORTED_PLATFORMS if x.name() == name), None) + + +def stable_path(path: str) -> str: + if get_os() == 'windows': + return path.replace("\\", "/") + + return path.replace("\\", "/") diff --git a/pyfastogt/utils.py b/pyfastogt/utils.py new file mode 100644 index 0000000..8bdd054 --- /dev/null +++ b/pyfastogt/utils.py @@ -0,0 +1,178 @@ +import errno +import os +import re +import shutil +import subprocess +import tarfile +import json +import ssl +import certifi +from validate_email import validate_email +from urllib.request import urlopen + + +class CommonError(Exception): + def __init__(self, value): + self.value_ = value + + def __str__(self): + return self.value_ + + +def is_valid_email(email: str, check_mx: bool) -> bool: + dns_valid = validate_email(email, check_mx=check_mx) + if not dns_valid: + return False + + validate_url = 'https://open.kickbox.com/v1/disposable/' + email + context = ssl._create_unverified_context() + response = urlopen(validate_url, context=context) + if response.status != 200: + return False + + data = response.read() + json_object = json.loads(data.decode("utf-8")) + is_disposable = json_object['disposable'] + return not is_disposable + + +def is_role_based_email(email: str) -> bool: + r = re.compile('([^@]+)@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,12})$') + match = r.match(email) + if not match: + return False + + start = match.group(1) + for x in ['noreply', 'support', 'admin', 'postmaster']: + if start == x: + return True + + return False + + +def read_file_line_by_line_to_list(file) -> list: + if not os.path.exists(file): + raise CommonError('file path: {0} not exists'.format(file)) + + file_array = [] + with open(file, "r") as ins: + for line in ins: + file_array.append(line.strip()) + + return file_array + + +def read_file_line_by_line_to_set(file) -> set: + if not os.path.exists(file): + raise CommonError('file path: {0} not exists'.format(file)) + + file_set = set() + with open(file, "r") as ins: + for line in ins: + file_set.add(line.strip()) + + return file_set + + +def download_file(url): + current_dir = os.getcwd() + file_name = url.split('/')[-1] + response = urlopen(url, cafile=certifi.where()) + if response.status != 200: + raise CommonError( + "Can't fetch url: {0}, status: {1}, response: {2}".format(url, response.status, response.reason)) + + f = open(file_name, 'wb') + file_size = 0 + header = response.getheader("Content-Length") + if header: + file_size = int(header) + + print("Downloading: {0} Bytes: {1}".format(file_name, file_size)) + + file_size_dl = 0 + block_sz = 8192 + while True: + buffer = response.read(block_sz) + if not buffer: + break + + file_size_dl += len(buffer) + f.write(buffer) + percent = 0 if not file_size else file_size_dl * 100. / file_size + status = r"%10d [%3.2f%%]" % (file_size_dl, percent) + status += chr(8) * (len(status) + 1) + print(status, end='\r') + + f.close() + return os.path.join(current_dir, file_name) + + +def extract_file(path, remove_after_extract=True): + current_dir = os.getcwd() + print("Extracting: {0}".format(path)) + try: + tar_file = tarfile.open(path) + except Exception as ex: + raise ex + + target_path = os.path.commonprefix(tar_file.getnames()) + try: + tar_file.extractall() + except Exception as ex: + raise ex + finally: + tar_file.close() + if remove_after_extract: + os.remove(path) + + return os.path.join(current_dir, target_path) + + +def git_clone(url: str, branch=None, remove_dot_git=True): + current_dir = os.getcwd() + if branch: + common_git_clone_line = ['git', 'clone', '--branch', branch, '--single-branch', url] + else: + common_git_clone_line = ['git', 'clone', '--depth=1', url] + cloned_dir_name = os.path.splitext(url.rsplit('/', 1)[-1])[0] + common_git_clone_line.append(cloned_dir_name) + subprocess.call(common_git_clone_line) + os.chdir(cloned_dir_name) + + common_git_clone_init_line = ['git', 'submodule', 'update', '--init', '--recursive'] + subprocess.call(common_git_clone_init_line) + directory = os.path.join(current_dir, cloned_dir_name) + if remove_dot_git: + shutil.rmtree(os.path.join(directory, '.git')) + os.chdir(current_dir) + return directory + + +def symlink_force(target, link_name): + try: + os.symlink(target, link_name) + except OSError as e: + if e.errno == errno.EEXIST: + os.remove(link_name) + os.symlink(target, link_name) + else: + raise e + + +# Search for number in array +def binary_search_impl(number, array, lo, hi): + if hi < lo: + return False + + mid = (lo + hi) // 2 + if number == array[mid]: + return True + elif number < array[mid]: + return binary_search_impl(number, array, lo, mid - 1) + else: + return binary_search_impl(number, array, mid + 1, hi) + + +def binary_search_number(anum, array): # convenience interface to binary_search() + return binary_search_impl(anum, array, 0, len(array) - 1) diff --git a/pyfastogt/verify_sign.py b/pyfastogt/verify_sign.py new file mode 100644 index 0000000..c8203a0 --- /dev/null +++ b/pyfastogt/verify_sign.py @@ -0,0 +1,74 @@ +import Crypto.Random +from Crypto.Hash import SHA +from Crypto.PublicKey import RSA +from Crypto.Signature import PKCS1_v1_5 + + +class Reader(object): + def __init__(self, file_path): + self.file_path_ = file_path + + def read(self, format='PEM'): + private_key_file = open(self.file_path_, 'rb') + private_key = RSA.importKey(private_key_file.read()) + public_key = private_key.publickey() + return private_key.exportKey(format), public_key.exportKey(format) + + +def write_key(file_path, key_data): + key_file = open(file_path, 'wb') + key_file.write(key_data) + key_file.close() + + +class Generator(object): + def __init__(self, bits_length=1024): + self.bits_length_ = bits_length + + def generate(self, format='PEM'): + random_gen = Crypto.Random.new().read + private_key = RSA.generate(self.bits_length_, random_gen) + public_key = private_key.publickey() + return private_key.exportKey(format), public_key.exportKey(format) + + +class Writer(object): + def __init__(self, file_path): + self.file_path_ = file_path + + def write(self, key_data): + return write_key(self.file_path_, key_data) + + +class Verify(object): + def __init__(self, public_key: str): + self.public_key_ = public_key + + def public_key(self) -> str: + return self.public_key_ + + def verify(self, data: bytes, signature: str) -> bool: + """ + Check that the provided signature corresponds to data + signed by the public key + """ + public_key = RSA.importKey(self.public_key_) + verifier = PKCS1_v1_5.new(public_key) + + h = SHA.new(data) + return verifier.verify(h, signature) + + +class Sign(Verify): + def __init__(self, public_key: str, private_key: str): + Verify.__init__(self, public_key) + self.private_key_ = private_key + + def sign(self, data: bytes) -> str: + """ + Sign data with private key + """ + private_key = RSA.importKey(self.private_key_) + signer = PKCS1_v1_5.new(private_key) + h = SHA.new(data) + return signer.sign(h) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4f3b060 --- /dev/null +++ b/setup.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Note: To use the 'upload' functionality of this file, you must: +# $ pip install twine + +import io +import os +import sys +from shutil import rmtree + +from setuptools import find_packages, setup, Command + +# Package meta-data. +NAME = 'pyfastogt' +DESCRIPTION = 'FastoGT python files.' +URL = 'https://github.com/fastogt/pybuild_utils' +EMAIL = 'support@fastogt.com' +AUTHOR = 'Alexandr Topilski' +REQUIRES_PYTHON = '>=3.0.0' +VERSION = '1.0.0' + +# What packages are required for this module to be executed? +REQUIRED = [ + 'validate_email', + 'distro', + 'certifi' +] + +# The rest you shouldn't have to touch too much :) +# ------------------------------------------------ +# Except, perhaps the License and Trove Classifiers! +# If you do change the License, remember to change the Trove Classifier for that! + +here = os.path.abspath(os.path.dirname(__file__)) + +# Import the README and use it as the long-description. +# Note: this will only work if 'README.rst' is present in your MANIFEST.in file! +with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = '\n' + f.read() + +# Load the package's __version__.py module as a dictionary. +about = {} +if not VERSION: + with open(os.path.join(here, NAME, '__version__.py')) as f: + exec(f.read(), about) +else: + about['__version__'] = VERSION + + +class UploadCommand(Command): + """Support setup.py upload.""" + + description = 'Build and publish the package.' + user_options = [] + + @staticmethod + def status(s): + """Prints things in bold.""" + print('\033[1m{0}\033[0m'.format(s)) + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + self.status('Removing previous builds…') + rmtree(os.path.join(here, 'dist')) + except OSError: + pass + + self.status('Building Source and Wheel (universal) distribution…') + os.system('{0} setup.py sdist bdist_wheel --universal'.format(sys.executable)) + + self.status('Uploading the package to PyPi via Twine…') + os.system('twine upload dist/*') + + self.status('Pushing git tags…') + os.system('git tag v{0}'.format(about['__version__'])) + os.system('git push --tags') + + sys.exit() + + +# Where the magic happens: +setup( + name=NAME, + version=about['__version__'], + description=DESCRIPTION, + long_description=long_description, + author=AUTHOR, + author_email=EMAIL, + python_requires=REQUIRES_PYTHON, + url=URL, + packages=find_packages(exclude=('tests',)), + # If your package is a single module, use this instead of 'packages': + # py_modules=['mypackage'], + + # entry_points={ + # 'console_scripts': ['mycli=mymodule:cli'], + # }, + install_requires=REQUIRED, + include_package_data=True, + license='LGPL', + classifiers=[ + # Trove classifiers + # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.0', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy' + ], + # $ setup.py publish support. + cmdclass={ + 'upload': UploadCommand, + }, +)