diff --git a/README.md b/README.md index 77eadd3..8e8a5ca 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,13 @@ $ git clone https://github.com/iganeshk/standardnotes-extensions.git $ cd standardnotes-extensions $ pip3 install -r requirements.txt ``` +* Visit the following link to generate a personal access token: +``` +$ https://github.com/settings/tokens +``` +![Github Personal Access Token](../assets/github_personal_token.png?raw=true) -* Use the env.sample to create a .env file for your environment variables. The utility will automatically load these when it starts. +* Use the env.sample to create a .env file for your environment variables and make sure you have placed your personal access token in the "token" attribute ``` # Sample ENV setup Variables (YAML) @@ -123,7 +128,7 @@ $ docker build -t standardnotes-extensions . # # Custom headers and headers various browsers *should* be OK with but aren't # - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,X-Application-Version,X-SNJS-Version'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; # # Tell client that this pre-flight info is valid for 20 days # @@ -135,13 +140,13 @@ $ docker build -t standardnotes-extensions . if ($request_method = 'POST') { add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,X-Application-Version,X-SNJS-Version'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } if ($request_method = 'GET') { add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,X-Application-Version,X-SNJS-Version'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } } diff --git a/build_repo.py b/build_repo.py index 59365c3..5ca23ea 100644 --- a/build_repo.py +++ b/build_repo.py @@ -17,25 +17,38 @@ import os import json import shutil from zipfile import ZipFile +from socket import gethostname as getlocalhostname import requests import yaml +LOCAL_HOSTNAME = getlocalhostname() def get_environment(base_dir): """ Parse the environment variables from .env """ - temp_envvar = yaml.load(""" - domain: https://domain.com/extensions + temp_env_var = yaml.load(""" github: username: token: + public_dir: public + extensions_dir: extensions + domain: https://domain.com/extensions + stdnotes_extensions_list: standardnotes-extensions-list.txt """, Loader=yaml.FullLoader) if os.path.isfile(os.path.join(base_dir, ".env")): with open(os.path.join(base_dir, ".env")) as temp_env_file: - temp_envvar = yaml.load(temp_env_file, Loader=yaml.FullLoader) + env_var = yaml.load(temp_env_file, Loader=yaml.FullLoader) - return temp_envvar + # if user hasn't updated the env, copy defaults to yaml dictionary + for key in temp_env_var: + try: + if not env_var[key]: + env_var[key] = temp_env_var[key] + except KeyError as e: + env_var[key] = temp_env_var[key] + + return env_var def process_zipball(repo_dir, release_version): @@ -70,12 +83,12 @@ def process_zipball(repo_dir, release_version): os.remove(os.path.join(repo_dir, release_version) + ".zip") -def git_clone_method(ext_yaml, public_dir, ext_has_update): +def git_clone_method(ext_yaml, public_path, ext_has_update): """ Get the latest repository and parse for metadata """ repo_name = ext_yaml['github'].split('/')[-1] - repo_dir = os.path.join(public_dir, repo_name) + repo_dir = os.path.join(public_path, repo_name) run([ 'git', 'clone', 'https://github.com/{github}.git'.format(**ext_yaml), '--quiet', '{}_tmp'.format(repo_name) @@ -83,7 +96,7 @@ def git_clone_method(ext_yaml, public_dir, ext_has_update): check=True) ext_last_commit = (run([ 'git', '--git-dir=' + - os.path.join(public_dir, '{}_tmp'.format(repo_name), '.git'), + os.path.join(public_path, '{}_tmp'.format(repo_name), '.git'), 'rev-list', '--tags', '--max-count=1' ], stdout=PIPE, @@ -91,7 +104,7 @@ def git_clone_method(ext_yaml, public_dir, ext_has_update): "\n", "")) ext_version = run([ 'git', '--git-dir', - os.path.join(public_dir, '{}_tmp'.format(repo_name), '.git'), + os.path.join(public_path, '{}_tmp'.format(repo_name), '.git'), 'describe', '--tags', ext_last_commit ], stdout=PIPE, @@ -101,58 +114,55 @@ def git_clone_method(ext_yaml, public_dir, ext_has_update): if not os.path.exists(os.path.join(repo_dir, ext_version)): ext_has_update = True shutil.move( - os.path.join(public_dir, '{}_tmp'.format(repo_name)), - os.path.join(public_dir, repo_name, '{}'.format(ext_version))) + os.path.join(public_path, '{}_tmp'.format(repo_name)), + os.path.join(public_path, repo_name, '{}'.format(ext_version))) # Delete .git resource from the directory shutil.rmtree( - os.path.join(public_dir, repo_name, '{}'.format(ext_version), + os.path.join(public_path, repo_name, '{}'.format(ext_version), '.git')) else: # ext already up-to-date # print('Extension: {} - {} (already up-to-date)'.format(ext_yaml['name'], ext_version)) # clean-up - shutil.rmtree(os.path.join(public_dir, '{}_tmp'.format(repo_name))) + shutil.rmtree(os.path.join(public_path, '{}_tmp'.format(repo_name))) return ext_version, ext_has_update -def parse_extensions(base_dir, base_url, ghub_session): +def parse_extensions(base_dir, extensions_dir, public_dir, base_url, stdnotes_ext_list_path, ghub_headers): """ Build Standard Notes extensions repository using Github meta-data """ - - extension_dir = os.path.join(base_dir, 'extensions') - public_dir = os.path.join(base_dir, 'public') - if not os.path.exists(os.path.join(public_dir)): - os.makedirs(public_dir) - os.chdir(public_dir) + extension_path = extensions_dir + public_path = public_dir + os.chdir(public_path) extensions = [] + std_ext_list = [] + std_ext_list = parse_stdnotes_extensions(stdnotes_ext_list_path) # Get all extensions, sort extensions alphabetically along by their by type - extfiles = [x for x in sorted(os.listdir(extension_dir)) if not x.endswith('theme.yaml') and x.endswith('.yaml')] - themefiles = [x for x in sorted(os.listdir(extension_dir)) if x.endswith('theme.yaml')] + extfiles = [x for x in sorted(os.listdir(extension_path)) if not x.endswith('theme.yaml') and x.endswith('.yaml')] + themefiles = [y for y in sorted(os.listdir(extension_path)) if y.endswith('theme.yaml')] extfiles.extend(themefiles) for extfile in extfiles: - with open(os.path.join(extension_dir, extfile)) as extyaml: + with open(os.path.join(extension_path, extfile)) as extyaml: ext_yaml = yaml.load(extyaml, Loader=yaml.FullLoader) ext_has_update = False repo_name = ext_yaml['github'].split('/')[-1] - repo_dir = os.path.join(public_dir, repo_name) - - # If we don't have a Github API Session, do git-clone instead - if ghub_session is not None: + repo_dir = os.path.join(public_path, repo_name) + # If we have valid github personal access token + if ghub_headers: # Get extension's github release meta-data ext_git_info = json.loads( - ghub_session.get( + requests.get( 'https://api.github.com/repos/{github}/releases/latest'. - format(**ext_yaml)).text) + format(**ext_yaml), headers=ghub_headers).text) try: ext_version = ext_git_info['tag_name'] except KeyError: - # No release's found - print( - "Error: Unable to update %s (%s) does it have a release at Github?" - % (ext_yaml['name'], extfile)) + # No github releases found + print('Skipping: {:38s}\t(github repository not found)'.format( + ext_yaml['name'])) continue # Check if extension directory already exists if not os.path.exists(repo_dir): @@ -162,7 +172,7 @@ def parse_extensions(base_dir, base_url, ghub_session): ext_has_update = True os.makedirs(os.path.join(repo_dir, ext_version)) # Grab the release and then unpack it - with requests.get(ext_git_info['zipball_url'], + with requests.get(ext_git_info['zipball_url'], headers=ghub_headers, stream=True) as zipball_stream: with open( os.path.join(repo_dir, ext_version) + ".zip", @@ -172,7 +182,11 @@ def parse_extensions(base_dir, base_url, ghub_session): process_zipball(repo_dir, ext_version) else: ext_version, ext_has_update = git_clone_method( - ext_yaml, public_dir, ext_has_update) + ext_yaml, public_path, ext_has_update) + + if extfile in std_ext_list: + ext_id = ext_yaml['id'].rsplit('.', 1)[1] + ext_yaml['id'] = '%s.%s' % (LOCAL_HOSTNAME, ext_id) # Build extension info (stateless) # https://domain.com/sub-domain/my-extension/index.json @@ -199,33 +213,33 @@ def parse_extensions(base_dir, base_url, ghub_session): # Strip empty values extension = {k: v for k, v in extension.items() if v} - # Check if extension is already up-to-date () + # Check if extension is already up-to-date if ext_has_update: # Generate JSON file for each extension - with open(os.path.join(public_dir, repo_name, 'index.json'), + with open(os.path.join(public_path, repo_name, 'index.json'), 'w') as ext_json: json.dump(extension, ext_json, indent=4) if extfile.endswith("theme.yaml"): print('Theme: {:34s} {:6s}\t(updated)'.format( - ext_yaml['name'], ext_version)) + ext_yaml['name'], ext_version.strip('v'))) else: print('Extension: {:30s} {:6s}\t(updated)'.format( - ext_yaml['name'], ext_version)) + ext_yaml['name'], ext_version.strip('v'))) else: # ext already up-to-date if extfile.endswith("theme.yaml"): print('Theme: {:34s} {:6s}\t(already up-to-date)'.format( - ext_yaml['name'], ext_version)) + ext_yaml['name'], ext_version.strip('v'))) else: print('Extension: {:30s} {:6s}\t(already up-to-date)'.format( - ext_yaml['name'], ext_version)) + ext_yaml['name'], ext_version.strip('v'))) extensions.append(extension) os.chdir('..') # Generate the main repository index JSON # https://domain.com/sub-domain/my-index.json - with open(os.path.join(public_dir, 'index.json'), 'w') as ext_json: + with open(os.path.join(public_path, 'index.json'), 'w') as ext_json: json.dump( dict( content_type='SN|Repo', @@ -238,6 +252,27 @@ def parse_extensions(base_dir, base_url, ghub_session): print("\nProcessed: {:20s}{} extensions. (Components: {}, Themes: {})".format("", len(extfiles), len(extfiles)-len(themefiles), len(themefiles))) print("Repository Endpoint URL: {:6s}{}/index.json".format("", base_url)) +def parse_stdnotes_extensions(stdnotes_ext_list_path): + """ + To circumvent around the issue: https://github.com/standardnotes/desktop/issues/789 + We'll be parsing standard note's extensions package ids with local hostname followed + by package name + """ + if not os.path.exists(stdnotes_ext_list_path): + print("\n⚠️ WARNING: Unable to locate standard notes extensions list file, make sure you've \ + cloned the source repository properly\ + ") + print("You may encounter issues registering extensions, checkout ") + print("https://github.com/standardnotes/desktop/issues/789 for more details\n") + else: + std_exts_list = [] + with open(stdnotes_ext_list_path) as list_file: + for line in list_file: + if not line.startswith('#'): + std_exts_list.append(line.rstrip()) + return std_exts_list + + def main(): """ teh main function @@ -246,38 +281,51 @@ def main(): # Get environment variables env_var = get_environment(base_dir) base_url = env_var['domain'] - while base_url.endswith('/'): - base_url = base_url[:-1] + extensions_dir = env_var['extensions_dir'] + if os.path.exists(os.path.join(base_dir, extensions_dir)): + extensions_dir = os.path.join(base_dir, extensions_dir) + else: + print("\n⚠️ WARNING: Unable to locate extensions directory, make sure you've \ + cloned the source repository properly and try again") + sys.exit(1) + public_dir = env_var['public_dir'] + if os.path.exists(os.path.join(base_dir, public_dir)): + public_dir = os.path.join(base_dir, public_dir) + else: + os.makedirs(os.path.join(base_dir, public_dir)) + public_dir = os.path.join(base_dir, public_dir) - if (env_var['github']['username'] and env_var['github']['token']): + stdnotes_ext_list = env_var['stdnotes_extensions_list'] + stdnotes_ext_list_path = os.path.join(base_dir, stdnotes_ext_list) + ghub_auth_complete = False + ghub_headers = False + + if env_var['github']['token']: # Get a re-usable session object using user credentials - ghub_session = requests.Session() - ghub_session.auth = (env_var['github']['username'], - env_var['github']['token']) + ghub_headers = {'Authorization': f'token %s' % env_var['github']['token']} try: - ghub_verify = ghub_session.get("https://api.github.com/") - if not ghub_verify.headers['status'] == "200 OK": - print("Error: %s " % ghub_verify.headers['status']) + ghub_verify = requests.get("https://api.github.com/", headers=ghub_headers) + if not ghub_verify.status_code == 200: + print("ERROR: %s " % ghub_verify.headers['status']) print( "Bad Github credentials in the .env file, check and try again." ) sys.exit(1) + ghub_auth_complete = True except Exception as e: - print("Unknown error occurred: %s" % e) - # Build extensions - parse_extensions(base_dir, base_url, ghub_session) - # Terminate Session - ghub_session.close() - else: + print("ERROR: %s" % e) + + if not ghub_auth_complete: # Environment file missing print( - "Environment variables not set (have a look at env.sample). Using Git Clone method instead" + "Environment variables not set (have a look at env.sample). Using git-clone method instead" ) input( - "⚠️ this is an in-efficient process, Press any key to continue:\n") - parse_extensions(base_dir, base_url, None) - sys.exit(0) + "⚠️ WARNING: This is an in-efficient process, press any key to go ahead anyway:\n") + # Build extensions + parse_extensions(base_dir, extensions_dir, public_dir, base_url, stdnotes_ext_list_path, ghub_headers) + sys.exit(0) if __name__ == '__main__': main() diff --git a/env.sample b/env.sample index d066bb6..5ab4007 100644 --- a/env.sample +++ b/env.sample @@ -3,14 +3,23 @@ # # $ cp env.sample .env # +# WARNING # Do not include this new file in source control # Github Credentials -# Generate your token here: https://github.com/settings/tokens +# Generate your personal access token here: https://github.com/settings/tokens # No additional permission required, this is just to avoid github api rate limits -# - -domain: https://domain.com/extensions - github: username: USERNAME token: TOKEN + +# EXTENSION PUBLICATION DOMAIN +domain: https://domain.com/extensions + +# EXTENSIONS DIRECTORY +extensions_dir: extensions + +# EXTENSIONS PUBLICATION DIRECTORY +public_dir: public + +# STANDARD HOSTS EXTENSIONS LIST +stdnotes_extensions_list: standardnotes-extensions-list.txt diff --git a/extensions/subtle-dark-theme.yaml b/extensions/subtle-dark-theme.yaml index 1dda429..7ee0320 100644 --- a/extensions/subtle-dark-theme.yaml +++ b/extensions/subtle-dark-theme.yaml @@ -3,7 +3,7 @@ id: tech.gunderson.sn-theme-subtle-dark github: Parkertg/sn-theme-subtle-dark main: main.css -name: Subtle Light +name: Subtle Dark content_type: SN|Theme area: themes version: 1.1 diff --git a/standardnotes-extensions-list.txt b/standardnotes-extensions-list.txt new file mode 100644 index 0000000..544e446 --- /dev/null +++ b/standardnotes-extensions-list.txt @@ -0,0 +1,32 @@ +############################################################################# +# List of standard notes's extensions who's identifier need to be modified to +# get around the issue: https://github.com/standardnotes/desktop/issues/789 +############################################################################# +action-bar.yaml +autobiography-theme.yaml +autocomplete-tags.yaml +bold-editor.yaml +code-editor.yaml +focus-theme.yaml +folders-component.yaml +futura-theme.yaml +github-push.yaml +grey-scale-theme.yaml +gruvbox-muted-theme-blue.yaml +markdown-basic.yaml +markdown-pro-editor.yaml +math-editor.yaml +mfa-link.yaml +midnight-theme.yaml +minimal-markdown-editor.yaml +no-distraction-theme.yaml +plus-editor.yaml +secure-spreadsheets.yaml +simple-task-editor.yaml +slate-theme.yaml +standard-gray-theme.yaml +subtle-dark-theme.yaml +subtle-light-theme.yaml +titanium-theme.yaml +token-vault.yaml +vim-editor.yaml