standardnotes/standardnotes-extensions2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Dec 2021 Update

Browse Source 
Switched to newer Github API standards
Parse Standard notes extensions identifiers to bypass subscription issue
Minor code changes
This commit is contained in:
iganeshk
2021-12-29 15:40:54 -05:00
parent a44c07e616
commit fb0463a528
5 changed files with 164 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@@ -24,8 +24,13 @@ $ git clone https://github.com/iganeshk/standardnotes-extensions.git
$ cd standardnotes-extensions
$ pip3 install -r requirements.txt
```
* Visit the following link to generate a personal access token:
```
$ https://github.com/settings/tokens
```
![Github Personal Access Token](../assets/github_personal_token.png?raw=true)
* Use the env.sample to create a .env file for your environment variables. The utility will automatically load these when it starts.
* Use the env.sample to create a .env file for your environment variables and make sure you have placed your personal access token in the "token" attribute
```
# Sample ENV setup Variables (YAML)
@@ -123,7 +128,7 @@ $ docker build -t standardnotes-extensions .
#
# Custom headers and headers various browsers *should* be OK with but aren't
#
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,X-Application-Version,X-SNJS-Version';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
#
# Tell client that this pre-flight info is valid for 20 days
#
@@ -135,13 +140,13 @@ $ docker build -t standardnotes-extensions .
if ($request_method = 'POST') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,X-Application-Version,X-SNJS-Version';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
}
if ($request_method = 'GET') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,X-Application-Version,X-SNJS-Version';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
}
}

168
build_repo.py
View File

@@ -17,25 +17,38 @@ import os
import json
import shutil
from zipfile import ZipFile
from socket import gethostname as getlocalhostname
import requests
import yaml
LOCAL_HOSTNAME = getlocalhostname()
def get_environment(base_dir):
"""
Parse the environment variables from .env
"""
temp_envvar = yaml.load("""
domain: https://domain.com/extensions
temp_env_var = yaml.load("""
github:
username:
token:
public_dir: public
extensions_dir: extensions
domain: https://domain.com/extensions
stdnotes_extensions_list: standardnotes-extensions-list.txt
""", Loader=yaml.FullLoader)
if os.path.isfile(os.path.join(base_dir, ".env")):
with open(os.path.join(base_dir, ".env")) as temp_env_file:
temp_envvar = yaml.load(temp_env_file, Loader=yaml.FullLoader)
env_var = yaml.load(temp_env_file, Loader=yaml.FullLoader)
return temp_envvar
# if user hasn't updated the env, copy defaults to yaml dictionary
for key in temp_env_var:
try:
if not env_var[key]:
env_var[key] = temp_env_var[key]
except KeyError as e:
env_var[key] = temp_env_var[key]
return env_var
def process_zipball(repo_dir, release_version):
@@ -70,12 +83,12 @@ def process_zipball(repo_dir, release_version):
os.remove(os.path.join(repo_dir, release_version) + ".zip")
def git_clone_method(ext_yaml, public_dir, ext_has_update):
def git_clone_method(ext_yaml, public_path, ext_has_update):
"""
Get the latest repository and parse for metadata
"""
repo_name = ext_yaml['github'].split('/')[-1]
repo_dir = os.path.join(public_dir, repo_name)
repo_dir = os.path.join(public_path, repo_name)
run([
'git', 'clone', 'https://github.com/{github}.git'.format(**ext_yaml),
'--quiet', '{}_tmp'.format(repo_name)
@@ -83,7 +96,7 @@ def git_clone_method(ext_yaml, public_dir, ext_has_update):
check=True)
ext_last_commit = (run([
'git', '--git-dir=' +
os.path.join(public_dir, '{}_tmp'.format(repo_name), '.git'),
os.path.join(public_path, '{}_tmp'.format(repo_name), '.git'),
'rev-list', '--tags', '--max-count=1'
],
stdout=PIPE,
@@ -91,7 +104,7 @@ def git_clone_method(ext_yaml, public_dir, ext_has_update):
"\n", ""))
ext_version = run([
'git', '--git-dir',
os.path.join(public_dir, '{}_tmp'.format(repo_name), '.git'),
os.path.join(public_path, '{}_tmp'.format(repo_name), '.git'),
'describe', '--tags', ext_last_commit
],
stdout=PIPE,
@@ -101,58 +114,55 @@ def git_clone_method(ext_yaml, public_dir, ext_has_update):
if not os.path.exists(os.path.join(repo_dir, ext_version)):
ext_has_update = True
shutil.move(
os.path.join(public_dir, '{}_tmp'.format(repo_name)),
os.path.join(public_dir, repo_name, '{}'.format(ext_version)))
os.path.join(public_path, '{}_tmp'.format(repo_name)),
os.path.join(public_path, repo_name, '{}'.format(ext_version)))
# Delete .git resource from the directory
shutil.rmtree(
os.path.join(public_dir, repo_name, '{}'.format(ext_version),
os.path.join(public_path, repo_name, '{}'.format(ext_version),
'.git'))
else:
# ext already up-to-date
# print('Extension: {} - {} (already up-to-date)'.format(ext_yaml['name'], ext_version))
# clean-up
shutil.rmtree(os.path.join(public_dir, '{}_tmp'.format(repo_name)))
shutil.rmtree(os.path.join(public_path, '{}_tmp'.format(repo_name)))
return ext_version, ext_has_update
def parse_extensions(base_dir, base_url, ghub_session):
def parse_extensions(base_dir, extensions_dir, public_dir, base_url, stdnotes_ext_list_path, ghub_headers):
"""
Build Standard Notes extensions repository using Github meta-data
"""
extension_dir = os.path.join(base_dir, 'extensions')
public_dir = os.path.join(base_dir, 'public')
if not os.path.exists(os.path.join(public_dir)):
os.makedirs(public_dir)
os.chdir(public_dir)
extension_path = extensions_dir
public_path = public_dir
os.chdir(public_path)
extensions = []
std_ext_list = []
std_ext_list = parse_stdnotes_extensions(stdnotes_ext_list_path)
# Get all extensions, sort extensions alphabetically along by their by type
extfiles = [x for x in sorted(os.listdir(extension_dir)) if not x.endswith('theme.yaml') and x.endswith('.yaml')]
themefiles = [x for x in sorted(os.listdir(extension_dir)) if x.endswith('theme.yaml')]
extfiles = [x for x in sorted(os.listdir(extension_path)) if not x.endswith('theme.yaml') and x.endswith('.yaml')]
themefiles = [y for y in sorted(os.listdir(extension_path)) if y.endswith('theme.yaml')]
extfiles.extend(themefiles)
for extfile in extfiles:
with open(os.path.join(extension_dir, extfile)) as extyaml:
with open(os.path.join(extension_path, extfile)) as extyaml:
ext_yaml = yaml.load(extyaml, Loader=yaml.FullLoader)
ext_has_update = False
repo_name = ext_yaml['github'].split('/')[-1]
repo_dir = os.path.join(public_dir, repo_name)
# If we don't have a Github API Session, do git-clone instead
if ghub_session is not None:
repo_dir = os.path.join(public_path, repo_name)
# If we have valid github personal access token
if ghub_headers:
# Get extension's github release meta-data
ext_git_info = json.loads(
ghub_session.get(
requests.get(
'https://api.github.com/repos/{github}/releases/latest'.
format(**ext_yaml)).text)
format(**ext_yaml), headers=ghub_headers).text)
try:
ext_version = ext_git_info['tag_name']
except KeyError:
# No release's found
print(
"Error: Unable to update %s (%s) does it have a release at Github?"
% (ext_yaml['name'], extfile))
# No github releases found
print('Skipping: {:38s}\t(github repository not found)'.format(
ext_yaml['name']))
continue
# Check if extension directory already exists
if not os.path.exists(repo_dir):
@@ -162,7 +172,7 @@ def parse_extensions(base_dir, base_url, ghub_session):
ext_has_update = True
os.makedirs(os.path.join(repo_dir, ext_version))
# Grab the release and then unpack it
with requests.get(ext_git_info['zipball_url'],
with requests.get(ext_git_info['zipball_url'], headers=ghub_headers,
stream=True) as zipball_stream:
with open(
os.path.join(repo_dir, ext_version) + ".zip",
@@ -172,7 +182,11 @@ def parse_extensions(base_dir, base_url, ghub_session):
process_zipball(repo_dir, ext_version)
else:
ext_version, ext_has_update = git_clone_method(
ext_yaml, public_dir, ext_has_update)
ext_yaml, public_path, ext_has_update)
if extfile in std_ext_list:
ext_id = ext_yaml['id'].rsplit('.', 1)[1]
ext_yaml['id'] = '%s.%s' % (LOCAL_HOSTNAME, ext_id)
# Build extension info (stateless)
# https://domain.com/sub-domain/my-extension/index.json
@@ -199,33 +213,33 @@ def parse_extensions(base_dir, base_url, ghub_session):
# Strip empty values
extension = {k: v for k, v in extension.items() if v}
# Check if extension is already up-to-date ()
# Check if extension is already up-to-date
if ext_has_update:
# Generate JSON file for each extension
with open(os.path.join(public_dir, repo_name, 'index.json'),
with open(os.path.join(public_path, repo_name, 'index.json'),
'w') as ext_json:
json.dump(extension, ext_json, indent=4)
if extfile.endswith("theme.yaml"):
print('Theme: {:34s} {:6s}\t(updated)'.format(
ext_yaml['name'], ext_version))
ext_yaml['name'], ext_version.strip('v')))
else:
print('Extension: {:30s} {:6s}\t(updated)'.format(
ext_yaml['name'], ext_version))
ext_yaml['name'], ext_version.strip('v')))
else:
# ext already up-to-date
if extfile.endswith("theme.yaml"):
print('Theme: {:34s} {:6s}\t(already up-to-date)'.format(
ext_yaml['name'], ext_version))
ext_yaml['name'], ext_version.strip('v')))
else:
print('Extension: {:30s} {:6s}\t(already up-to-date)'.format(
ext_yaml['name'], ext_version))
ext_yaml['name'], ext_version.strip('v')))
extensions.append(extension)
os.chdir('..')
# Generate the main repository index JSON
# https://domain.com/sub-domain/my-index.json
with open(os.path.join(public_dir, 'index.json'), 'w') as ext_json:
with open(os.path.join(public_path, 'index.json'), 'w') as ext_json:
json.dump(
dict(
content_type='SN|Repo',
@@ -238,6 +252,27 @@ def parse_extensions(base_dir, base_url, ghub_session):
print("\nProcessed: {:20s}{} extensions. (Components: {}, Themes: {})".format("", len(extfiles), len(extfiles)-len(themefiles), len(themefiles)))
print("Repository Endpoint URL: {:6s}{}/index.json".format("", base_url))
def parse_stdnotes_extensions(stdnotes_ext_list_path):
"""
To circumvent around the issue: https://github.com/standardnotes/desktop/issues/789
We'll be parsing standard note's extensions package ids with local hostname followed
by package name
"""
if not os.path.exists(stdnotes_ext_list_path):
print("\n⚠️ WARNING: Unable to locate standard notes extensions list file, make sure you've \
cloned the source repository properly\
")
print("You may encounter issues registering extensions, checkout ")
print("https://github.com/standardnotes/desktop/issues/789 for more details\n")
else:
std_exts_list = []
with open(stdnotes_ext_list_path) as list_file:
for line in list_file:
if not line.startswith('#'):
std_exts_list.append(line.rstrip())
return std_exts_list
def main():
"""
teh main function
@@ -246,38 +281,51 @@ def main():
# Get environment variables
env_var = get_environment(base_dir)
base_url = env_var['domain']
while base_url.endswith('/'):
base_url = base_url[:-1]
extensions_dir = env_var['extensions_dir']
if os.path.exists(os.path.join(base_dir, extensions_dir)):
extensions_dir = os.path.join(base_dir, extensions_dir)
else:
print("\n⚠️ WARNING: Unable to locate extensions directory, make sure you've \
cloned the source repository properly and try again")
sys.exit(1)
public_dir = env_var['public_dir']
if os.path.exists(os.path.join(base_dir, public_dir)):
public_dir = os.path.join(base_dir, public_dir)
else:
os.makedirs(os.path.join(base_dir, public_dir))
public_dir = os.path.join(base_dir, public_dir)
if (env_var['github']['username'] and env_var['github']['token']):
stdnotes_ext_list = env_var['stdnotes_extensions_list']
stdnotes_ext_list_path = os.path.join(base_dir, stdnotes_ext_list)
ghub_auth_complete = False
ghub_headers = False
if env_var['github']['token']:
# Get a re-usable session object using user credentials
ghub_session = requests.Session()
ghub_session.auth = (env_var['github']['username'],
env_var['github']['token'])
ghub_headers = {'Authorization': f'token %s' % env_var['github']['token']}
try:
ghub_verify = ghub_session.get("https://api.github.com/")
if not ghub_verify.headers['status'] == "200 OK":
print("Error: %s " % ghub_verify.headers['status'])
ghub_verify = requests.get("https://api.github.com/", headers=ghub_headers)
if not ghub_verify.status_code == 200:
print("ERROR: %s " % ghub_verify.headers['status'])
print(
"Bad Github credentials in the .env file, check and try again."
)
sys.exit(1)
ghub_auth_complete = True
except Exception as e:
print("Unknown error occurred: %s" % e)
# Build extensions
parse_extensions(base_dir, base_url, ghub_session)
# Terminate Session
ghub_session.close()
else:
print("ERROR: %s" % e)
if not ghub_auth_complete:
# Environment file missing
print(
"Environment variables not set (have a look at env.sample). Using Git Clone method instead"
"Environment variables not set (have a look at env.sample). Using git-clone method instead"
)
input(
"⚠️ this is an in-efficient process, Press any key to continue:\n")
parse_extensions(base_dir, base_url, None)
sys.exit(0)
"⚠️ WARNING: This is an in-efficient process, press any key to go ahead anyway:\n")
# Build extensions
parse_extensions(base_dir, extensions_dir, public_dir, base_url, stdnotes_ext_list_path, ghub_headers)
sys.exit(0)
if __name__ == '__main__':
main()

19
env.sample
View File

@@ -3,14 +3,23 @@
#
# $ cp env.sample .env
#
# WARNING
# Do not include this new file in source control
# Github Credentials
# Generate your token here: https://github.com/settings/tokens
# Generate your personal access token here: https://github.com/settings/tokens
# No additional permission required, this is just to avoid github api rate limits
#
domain: https://domain.com/extensions
github:
username: USERNAME
token: TOKEN
# EXTENSION PUBLICATION DOMAIN
domain: https://domain.com/extensions
# EXTENSIONS DIRECTORY
extensions_dir: extensions
# EXTENSIONS PUBLICATION DIRECTORY
public_dir: public
# STANDARD HOSTS EXTENSIONS LIST
stdnotes_extensions_list: standardnotes-extensions-list.txt

2
extensions/subtle-dark-theme.yaml
View File

@@ -3,7 +3,7 @@ id: tech.gunderson.sn-theme-subtle-dark
github: Parkertg/sn-theme-subtle-dark
main: main.css
name: Subtle Light
name: Subtle Dark
content_type: SN|Theme
area: themes
version: 1.1

32
standardnotes-extensions-list.txt Normal file
View File

@@ -0,0 +1,32 @@
#############################################################################
# List of standard notes's extensions who's identifier need to be modified to
# get around the issue: https://github.com/standardnotes/desktop/issues/789
#############################################################################
action-bar.yaml
autobiography-theme.yaml
autocomplete-tags.yaml
bold-editor.yaml
code-editor.yaml
focus-theme.yaml
folders-component.yaml
futura-theme.yaml
github-push.yaml
grey-scale-theme.yaml
gruvbox-muted-theme-blue.yaml
markdown-basic.yaml
markdown-pro-editor.yaml
math-editor.yaml
mfa-link.yaml
midnight-theme.yaml
minimal-markdown-editor.yaml
no-distraction-theme.yaml
plus-editor.yaml
secure-spreadsheets.yaml
simple-task-editor.yaml
slate-theme.yaml
standard-gray-theme.yaml
subtle-dark-theme.yaml
subtle-light-theme.yaml
titanium-theme.yaml
token-vault.yaml
vim-editor.yaml