diff --git a/.gitignore b/.gitignore index eb07033..7aad6eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ public +.env ### Python.gitignore diff --git a/README.md b/README.md index c3dfc76..86315b4 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ LaMetric-System-Monitor ## Standard Notes Extensions - Self-Hosted Repository -Host Standard Notes extensions on your own server. This utility parses list of extensions configured in YAML from the `\extensions` directory, builds a repository JSON index which can be plugged directly into Standard Notes Web/Desktop Clients. (https://standardnotes.org/) +Host Standard Notes extensions on your own server. This utility parses most of the open-source extensions available from original repository as well as other authors and builds a extensions repository which can be plugged directly into Standard Notes Web/Desktop Clients. (https://standardnotes.org/) + +Extensions are listed as YAML in the `\extensions` sub-directory, pull a request if you'd like to add yours. ### Requirements * Python 3 @@ -74,7 +76,8 @@ https://your-domain.com/extensions/index.json ``` ### Acknowledgments -This project was adapted from https://github.com/JokerQyou/snextensions to facilitate on-the-fly updating of extensions. +* This project was adapted from https://github.com/JokerQyou/snextensions to facilitate on-the-fly updating of extensions. +* Dracula Theme by https://github.com/cameronldn ### ToDo * Implement the usage of GitHub API for efficiency. \ No newline at end of file diff --git a/build_repo.py b/build_repo.py index 318eb0f..82ff771 100644 --- a/build_repo.py +++ b/build_repo.py @@ -11,21 +11,67 @@ public/ | | |-... <- other files |-index.json <- repo info, contain all extensions' info ''' +# from subprocess import run, PIPE +from zipfile import ZipFile import json import os import shutil -from subprocess import run, PIPE +import requests import yaml -def main(base_url): - ''' - main function - ''' - while base_url.endswith('/'): - base_url = base_url[:-1] +def get_environment(base_dir): + """ + Load .env file if present + """ + temp_envvar = yaml.load(""" + domain: https://domain.com/extensions + github: + username: + token: + """, Loader=yaml.FullLoader) + if os.path.isfile(os.path.join(base_dir, ".env")): + with open(os.path.join(base_dir, ".env")) as temp_env_file: + temp_envvar = yaml.load(temp_env_file, Loader=yaml.FullLoader) + return temp_envvar - base_dir = os.path.dirname(os.path.abspath(__file__)) + # Environment file missing + print("Please set your environment file (read env.sample)") + print("You might be rate limited while parsing extensions from Github, if you continue!") + input("Press any key to continue: ") + return temp_envvar + + +def process_zipball(repo_dir, release_version): + """ + Get release zipball and extract archive without the root directory + """ + with ZipFile(os.path.join(repo_dir, release_version) + ".zip", 'r') as zipball: + for member in zipball.namelist(): + # Parse files without root directory + filename = '/'.join(member.split('/')[1:]) + # Ignore the parent folder + if filename == '': continue + # Ignore dot files + if filename.startswith('.'): continue + source = zipball.open(member) + try: + target = open(os.path.join(repo_dir, release_version, filename), "wb") + with source, target: + target = open(os.path.join(repo_dir, release_version, filename), "wb") + shutil.copyfileobj(source, target) + except FileNotFoundError: + # Create the directory + os.makedirs(os.path.dirname(os.path.join(repo_dir, release_version, filename))) + continue + # Delete the archive zip + os.remove(os.path.join(repo_dir, release_version) + ".zip") + + +def parse_extensions(base_dir, base_url, ghub_session): + """ + Build Standard Notes extensions repository using Github meta-data + """ extension_dir = os.path.join(base_dir, 'extensions') public_dir = os.path.join(base_dir, 'public') if not os.path.exists(os.path.join(public_dir)): @@ -42,92 +88,112 @@ def main(base_url): with open(os.path.join(extension_dir, extfiles)) as extyaml: ext = yaml.load(extyaml, Loader=yaml.FullLoader) - # Build extension info + # Get extension Github meta-data + ext_git_info = json.loads(ghub_session.get('https://api.github.com/repos/{github}/releases/latest'.format(**ext)).text) + repo_name = ext['github'].split('/')[-1] - # https://example.com/sub-domain/my-extension/version/index.html - extension_url = '/'.join([base_url, repo_name, ext['main']]) - # https://example.com/sub-domain/my-extension/index.json - extension_info_url = '/'.join([base_url, repo_name, 'index.json']) - extension = dict( - identifier=ext['id'], - name=ext['name'], - content_type=ext['content_type'], - area=ext.get('area', None), - # supplying version not really a concern since it's checked for - version=ext['version'], - description=ext.get('description', None), - marketing_url=ext.get('marketing_url', None), - thumbnail_url=ext.get('thumbnail_url', None), - valid_until='2030-05-16T18:35:33.000Z', - url=extension_url, - download_url='https://github.com/{github}/archive/{version}.zip'. - format(**ext), - latest_url=extension_info_url, - flags=ext.get('flags', []), - dock_icon=ext.get('dock_icon', {}), - layerable=ext.get('layerable', None), - ) + repo_dir = os.path.join(public_dir, repo_name) - # Strip empty values - extension = {k: v for k, v in extension.items() if v} + # Check if extension directory alredy exists + if not os.path.exists(repo_dir): + os.makedirs(repo_dir) + # Check if extension with current release alredy exists + if not os.path.exists(os.path.join(repo_dir, ext_git_info['tag_name'])): + os.makedirs(os.path.join(repo_dir, ext_git_info['tag_name'])) + # Grab the release and then unpack it + with requests.get(ext_git_info['zipball_url'], stream=True) as r: + with open(os.path.join(repo_dir, ext_git_info['tag_name']) + ".zip", 'wb') as f: + shutil.copyfileobj(r.raw, f) + # unpack the zipball + process_zipball(repo_dir, ext_git_info['tag_name']) + # Build extension info + # https://example.com/sub-domain/my-extension/version/index.html + extension_url = '/'.join([base_url, repo_name, ext_git_info['tag_name'], ext['main']]) + # https://example.com/sub-domain/my-extension/index.json + extension_info_url = '/'.join([base_url, repo_name, 'index.json']) + extension = dict( + identifier=ext['id'], + name=ext['name'], + content_type=ext['content_type'], + area=ext.get('area', None), + version=ext_git_info['tag_name'], + description=ext.get('description', None), + marketing_url=ext.get('marketing_url', None), + thumbnail_url=ext.get('thumbnail_url', None), + valid_until='2030-05-16T18:35:33.000Z', + url=extension_url, + download_url='https://github.com/{github}/archive/{version}.zip'. + format(**ext), + latest_url=extension_info_url, + flags=ext.get('flags', []), + dock_icon=ext.get('dock_icon', {}), + layerable=ext.get('layerable', None), + statusBar=ext.get('statusBar', None), + ) - # Get the latest repository and parse for latest version - # TO-DO: Implement usage of Github API for efficiency - run([ - 'git', 'clone', 'https://github.com/{github}.git'.format(**ext), - '--quiet', '{}_temp'.format(repo_name) - ], - check=True) - ext_latest = (run([ - 'git', '--git-dir=' + - os.path.join(public_dir, '{}_temp'.format(repo_name), '.git'), - 'rev-list', '--tags', '--max-count=1' - ], - stdout=PIPE, - check=True).stdout.decode('utf-8').replace("\n", "")) - ext_latest_version = run([ - 'git', '--git-dir', - os.path.join(public_dir, '{}_temp'.format(repo_name), '.git'), - 'describe', '--tags', ext_latest - ], - stdout=PIPE, - check=True).stdout.decode('utf-8').replace( - "\n", "") + # Strip empty values + extension = {k: v for k, v in extension.items() if v} - # Tag the latest releases - extension['version'] = ext_latest_version - extension['url'] = '/'.join([ - base_url, repo_name, '{}'.format(ext_latest_version), ext['main'] - ]) - extension['download_url'] = ( - 'https://github.com/{}/archive/{}.zip'.format( - ext['github'], ext_latest_version)) + """ To-be deprecated Method + # Get the latest repository and parse for latest version + # TO-DO: Implement usage of Github API for efficiency - # check if latest version already exists - if not os.path.exists( - os.path.join(public_dir, repo_name, - '{}'.format(ext_latest_version))): - shutil.move( - os.path.join(public_dir, '{}_temp'.format(repo_name)), - os.path.join(public_dir, repo_name, - '{}'.format(ext_latest_version))) - # Delete .git resource from the directory - shutil.rmtree( - os.path.join(public_dir, repo_name, - '{}'.format(ext_latest_version), '.git')) - else: - # clean-up - shutil.rmtree(os.path.join(public_dir, - '{}_temp'.format(repo_name))) + run([ + 'git', 'clone', 'https://github.com/{github}.git'.format(**ext), + '--quiet', '{}_temp'.format(repo_name) + ], + check=True) + ext_latest = (run([ + 'git', '--git-dir=' + + os.path.join(public_dir, '{}_temp'.format(repo_name), '.git'), + 'rev-list', '--tags', '--max-count=1' + ], + stdout=PIPE, + check=True).stdout.decode('utf-8').replace("\n", "")) + ext_latest_version = run([ + 'git', '--git-dir', + os.path.join(public_dir, '{}_temp'.format(repo_name), '.git'), + 'describe', '--tags', ext_latest + ], + stdout=PIPE, + check=True).stdout.decode('utf-8').replace( + "\n", "") - # Generate JSON file for each extension - with open(os.path.join(public_dir, repo_name, 'index.json'), - 'w') as ext_json: - json.dump(extension, ext_json, indent=4) + # Tag the latest releases + extension['version'] = ext_latest_version + extension['url'] = '/'.join([ + base_url, repo_name, '{}'.format(ext_latest_version), ext['main'] + ]) + extension['download_url'] = ( + 'https://github.com/{}/archive/{}.zip'.format( + ext['github'], ext_latest_version)) - extensions.append(extension) - print('Loaded extension: {} - {}'.format(ext['name'], - ext_latest_version)) + # check if latest version already exists + if not os.path.exists( + os.path.join(public_dir, repo_name, + '{}'.format(ext_latest_version))): + shutil.move( + os.path.join(public_dir, '{}_temp'.format(repo_name)), + os.path.join(public_dir, repo_name, + '{}'.format(ext_latest_version))) + # Delete .git resource from the directory + shutil.rmtree( + os.path.join(public_dir, repo_name, + '{}'.format(ext_latest_version), '.git')) + else: + # clean-up + shutil.rmtree(os.path.join(public_dir, + '{}_temp'.format(repo_name))) + + """ + # Generate JSON file for each extension + with open(os.path.join(public_dir, repo_name, 'index.json'), + 'w') as ext_json: + json.dump(extension, ext_json, indent=4) + + extensions.append(extension) + print('Loaded extension: {} - {}'.format(ext['name'], + ext_git_info['tag_name'])) os.chdir('..') @@ -143,6 +209,35 @@ def main(base_url): indent=4, ) + # Terminate Session + ghub_session.close() + + +def main(): + """ + teh main function + """ + base_dir = os.path.dirname(os.path.abspath(__file__)) + # Get environment variables + env_var = get_environment(base_dir) + base_url = env_var['domain'] + while base_url.endswith('/'): + base_url = base_url[:-1] + + # Get a re-usable session object using user credentials + ghub_session = requests.Session() + ghub_session.auth = (env_var['github']['username'], env_var['github']['token']) + try: + ghub_verify = ghub_session.get("https://api.github.com/") + if not ghub_verify.headers['status'] == "200 OK": + print("Error: %s " % ghub_verify.headers['status']) + print("Bad Github credentials in the .env file, check and try again.") + exit(1) + except Exception as e: + print("Error %s" % e) + # Build extensions + parse_extensions(base_dir, base_url, ghub_session) if __name__ == '__main__': - main(os.getenv('URL', 'https://domain.com/extensions')) + # If URL variable + main() diff --git a/env.sample b/env.sample new file mode 100644 index 0000000..d066bb6 --- /dev/null +++ b/env.sample @@ -0,0 +1,16 @@ +# Sample ENV setup Variables (YAML) +# Copy this file and update as needed. +# +# $ cp env.sample .env +# +# Do not include this new file in source control +# Github Credentials +# Generate your token here: https://github.com/settings/tokens +# No additional permission required, this is just to avoid github api rate limits +# + +domain: https://domain.com/extensions + +github: + username: USERNAME + token: TOKEN diff --git a/requirements.txt b/requirements.txt index 4818cc5..0a36425 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -pyyaml \ No newline at end of file +pyyaml>=5.1.1 +requests>=2.22.0