standardnotes/standardnotes-extensions2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement Github API

Browse Source
This commit is contained in:
iganeshk
2020-01-17 14:51:50 -05:00
parent 1004c69bde
commit 146266bd24
5 changed files with 207 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,4 +1,5 @@
public
.env
### Python.gitignore

7
README.md
View File

@@ -1,7 +1,9 @@
<img alt="LaMetric-System-Monitor" src="https://standardnotes.org/assets/icon.png"/>
## Standard Notes Extensions - Self-Hosted Repository
Host Standard Notes extensions on your own server. This utility parses list of extensions configured in YAML from the `\extensions` directory, builds a repository JSON index which can be plugged directly into Standard Notes Web/Desktop Clients. (https://standardnotes.org/)
Host Standard Notes extensions on your own server. This utility parses most of the open-source extensions available from original repository as well as other authors and builds a extensions repository which can be plugged directly into Standard Notes Web/Desktop Clients. (https://standardnotes.org/)
Extensions are listed as YAML in the `\extensions` sub-directory, pull a request if you'd like to add yours.
### Requirements
* Python 3
@@ -74,7 +76,8 @@ https://your-domain.com/extensions/index.json
```
### Acknowledgments
This project was adapted from https://github.com/JokerQyou/snextensions to facilitate on-the-fly updating of extensions.
* This project was adapted from https://github.com/JokerQyou/snextensions to facilitate on-the-fly updating of extensions.
* Dracula Theme by https://github.com/cameronldn
### ToDo
* Implement the usage of GitHub API for efficiency.

271
build_repo.py
View File

@@ -11,21 +11,67 @@ public/
| | |-... <- other files
|-index.json <- repo info, contain all extensions' info
'''
# from subprocess import run, PIPE
from zipfile import ZipFile
import json
import os
import shutil
from subprocess import run, PIPE
import requests
import yaml
def main(base_url):
'''
main function
'''
while base_url.endswith('/'):
base_url = base_url[:-1]
def get_environment(base_dir):
"""
Load .env file if present
"""
temp_envvar = yaml.load("""
domain: https://domain.com/extensions
github:
username:
token:
""", Loader=yaml.FullLoader)
if os.path.isfile(os.path.join(base_dir, ".env")):
with open(os.path.join(base_dir, ".env")) as temp_env_file:
temp_envvar = yaml.load(temp_env_file, Loader=yaml.FullLoader)
return temp_envvar
base_dir = os.path.dirname(os.path.abspath(__file__))
# Environment file missing
print("Please set your environment file (read env.sample)")
print("You might be rate limited while parsing extensions from Github, if you continue!")
input("Press any key to continue: ")
return temp_envvar
def process_zipball(repo_dir, release_version):
"""
Get release zipball and extract archive without the root directory
"""
with ZipFile(os.path.join(repo_dir, release_version) + ".zip", 'r') as zipball:
for member in zipball.namelist():
# Parse files without root directory
filename = '/'.join(member.split('/')[1:])
# Ignore the parent folder
if filename == '': continue
# Ignore dot files
if filename.startswith('.'): continue
source = zipball.open(member)
try:
target = open(os.path.join(repo_dir, release_version, filename), "wb")
with source, target:
target = open(os.path.join(repo_dir, release_version, filename), "wb")
shutil.copyfileobj(source, target)
except FileNotFoundError:
# Create the directory
os.makedirs(os.path.dirname(os.path.join(repo_dir, release_version, filename)))
continue
# Delete the archive zip
os.remove(os.path.join(repo_dir, release_version) + ".zip")
def parse_extensions(base_dir, base_url, ghub_session):
"""
Build Standard Notes extensions repository using Github meta-data
"""
extension_dir = os.path.join(base_dir, 'extensions')
public_dir = os.path.join(base_dir, 'public')
if not os.path.exists(os.path.join(public_dir)):
@@ -42,92 +88,112 @@ def main(base_url):
with open(os.path.join(extension_dir, extfiles)) as extyaml:
ext = yaml.load(extyaml, Loader=yaml.FullLoader)
# Build extension info
# Get extension Github meta-data
ext_git_info = json.loads(ghub_session.get('https://api.github.com/repos/{github}/releases/latest'.format(**ext)).text)
repo_name = ext['github'].split('/')[-1]
# https://example.com/sub-domain/my-extension/version/index.html
extension_url = '/'.join([base_url, repo_name, ext['main']])
# https://example.com/sub-domain/my-extension/index.json
extension_info_url = '/'.join([base_url, repo_name, 'index.json'])
extension = dict(
identifier=ext['id'],
name=ext['name'],
content_type=ext['content_type'],
area=ext.get('area', None),
# supplying version not really a concern since it's checked for
version=ext['version'],
description=ext.get('description', None),
marketing_url=ext.get('marketing_url', None),
thumbnail_url=ext.get('thumbnail_url', None),
valid_until='2030-05-16T18:35:33.000Z',
url=extension_url,
download_url='https://github.com/{github}/archive/{version}.zip'.
format(**ext),
latest_url=extension_info_url,
flags=ext.get('flags', []),
dock_icon=ext.get('dock_icon', {}),
layerable=ext.get('layerable', None),
)
repo_dir = os.path.join(public_dir, repo_name)
# Strip empty values
extension = {k: v for k, v in extension.items() if v}
# Check if extension directory alredy exists
if not os.path.exists(repo_dir):
os.makedirs(repo_dir)
# Check if extension with current release alredy exists
if not os.path.exists(os.path.join(repo_dir, ext_git_info['tag_name'])):
os.makedirs(os.path.join(repo_dir, ext_git_info['tag_name']))
# Grab the release and then unpack it
with requests.get(ext_git_info['zipball_url'], stream=True) as r:
with open(os.path.join(repo_dir, ext_git_info['tag_name']) + ".zip", 'wb') as f:
shutil.copyfileobj(r.raw, f)
# unpack the zipball
process_zipball(repo_dir, ext_git_info['tag_name'])
# Build extension info
# https://example.com/sub-domain/my-extension/version/index.html
extension_url = '/'.join([base_url, repo_name, ext_git_info['tag_name'], ext['main']])
# https://example.com/sub-domain/my-extension/index.json
extension_info_url = '/'.join([base_url, repo_name, 'index.json'])
extension = dict(
identifier=ext['id'],
name=ext['name'],
content_type=ext['content_type'],
area=ext.get('area', None),
version=ext_git_info['tag_name'],
description=ext.get('description', None),
marketing_url=ext.get('marketing_url', None),
thumbnail_url=ext.get('thumbnail_url', None),
valid_until='2030-05-16T18:35:33.000Z',
url=extension_url,
download_url='https://github.com/{github}/archive/{version}.zip'.
format(**ext),
latest_url=extension_info_url,
flags=ext.get('flags', []),
dock_icon=ext.get('dock_icon', {}),
layerable=ext.get('layerable', None),
statusBar=ext.get('statusBar', None),
)
# Get the latest repository and parse for latest version
# TO-DO: Implement usage of Github API for efficiency
run([
'git', 'clone', 'https://github.com/{github}.git'.format(**ext),
'--quiet', '{}_temp'.format(repo_name)
],
check=True)
ext_latest = (run([
'git', '--git-dir=' +
os.path.join(public_dir, '{}_temp'.format(repo_name), '.git'),
'rev-list', '--tags', '--max-count=1'
],
stdout=PIPE,
check=True).stdout.decode('utf-8').replace("\n", ""))
ext_latest_version = run([
'git', '--git-dir',
os.path.join(public_dir, '{}_temp'.format(repo_name), '.git'),
'describe', '--tags', ext_latest
],
stdout=PIPE,
check=True).stdout.decode('utf-8').replace(
"\n", "")
# Strip empty values
extension = {k: v for k, v in extension.items() if v}
# Tag the latest releases
extension['version'] = ext_latest_version
extension['url'] = '/'.join([
base_url, repo_name, '{}'.format(ext_latest_version), ext['main']
])
extension['download_url'] = (
'https://github.com/{}/archive/{}.zip'.format(
ext['github'], ext_latest_version))
""" To-be deprecated Method
# Get the latest repository and parse for latest version
# TO-DO: Implement usage of Github API for efficiency
# check if latest version already exists
if not os.path.exists(
os.path.join(public_dir, repo_name,
'{}'.format(ext_latest_version))):
shutil.move(
os.path.join(public_dir, '{}_temp'.format(repo_name)),
os.path.join(public_dir, repo_name,
'{}'.format(ext_latest_version)))
# Delete .git resource from the directory
shutil.rmtree(
os.path.join(public_dir, repo_name,
'{}'.format(ext_latest_version), '.git'))
else:
# clean-up
shutil.rmtree(os.path.join(public_dir,
'{}_temp'.format(repo_name)))
run([
'git', 'clone', 'https://github.com/{github}.git'.format(**ext),
'--quiet', '{}_temp'.format(repo_name)
],
check=True)
ext_latest = (run([
'git', '--git-dir=' +
os.path.join(public_dir, '{}_temp'.format(repo_name), '.git'),
'rev-list', '--tags', '--max-count=1'
],
stdout=PIPE,
check=True).stdout.decode('utf-8').replace("\n", ""))
ext_latest_version = run([
'git', '--git-dir',
os.path.join(public_dir, '{}_temp'.format(repo_name), '.git'),
'describe', '--tags', ext_latest
],
stdout=PIPE,
check=True).stdout.decode('utf-8').replace(
"\n", "")
# Generate JSON file for each extension
with open(os.path.join(public_dir, repo_name, 'index.json'),
'w') as ext_json:
json.dump(extension, ext_json, indent=4)
# Tag the latest releases
extension['version'] = ext_latest_version
extension['url'] = '/'.join([
base_url, repo_name, '{}'.format(ext_latest_version), ext['main']
])
extension['download_url'] = (
'https://github.com/{}/archive/{}.zip'.format(
ext['github'], ext_latest_version))
extensions.append(extension)
print('Loaded extension: {} - {}'.format(ext['name'],
ext_latest_version))
# check if latest version already exists
if not os.path.exists(
os.path.join(public_dir, repo_name,
'{}'.format(ext_latest_version))):
shutil.move(
os.path.join(public_dir, '{}_temp'.format(repo_name)),
os.path.join(public_dir, repo_name,
'{}'.format(ext_latest_version)))
# Delete .git resource from the directory
shutil.rmtree(
os.path.join(public_dir, repo_name,
'{}'.format(ext_latest_version), '.git'))
else:
# clean-up
shutil.rmtree(os.path.join(public_dir,
'{}_temp'.format(repo_name)))
"""
# Generate JSON file for each extension
with open(os.path.join(public_dir, repo_name, 'index.json'),
'w') as ext_json:
json.dump(extension, ext_json, indent=4)
extensions.append(extension)
print('Loaded extension: {} - {}'.format(ext['name'],
ext_git_info['tag_name']))
os.chdir('..')
@@ -143,6 +209,35 @@ def main(base_url):
indent=4,
)
# Terminate Session
ghub_session.close()
def main():
"""
teh main function
"""
base_dir = os.path.dirname(os.path.abspath(__file__))
# Get environment variables
env_var = get_environment(base_dir)
base_url = env_var['domain']
while base_url.endswith('/'):
base_url = base_url[:-1]
# Get a re-usable session object using user credentials
ghub_session = requests.Session()
ghub_session.auth = (env_var['github']['username'], env_var['github']['token'])
try:
ghub_verify = ghub_session.get("https://api.github.com/")
if not ghub_verify.headers['status'] == "200 OK":
print("Error: %s " % ghub_verify.headers['status'])
print("Bad Github credentials in the .env file, check and try again.")
exit(1)
except Exception as e:
print("Error %s" % e)
# Build extensions
parse_extensions(base_dir, base_url, ghub_session)
if __name__ == '__main__':
main(os.getenv('URL', 'https://domain.com/extensions'))
# If URL variable
main()

16
env.sample Normal file
View File

@@ -0,0 +1,16 @@
# Sample ENV setup Variables (YAML)
# Copy this file and update as needed.
#
# $ cp env.sample .env
#
# Do not include this new file in source control
# Github Credentials
# Generate your token here: https://github.com/settings/tokens
# No additional permission required, this is just to avoid github api rate limits
#
domain: https://domain.com/extensions
github:
username: USERNAME
token: TOKEN

3
requirements.txt
View File

@@ -1 +1,2 @@
pyyaml
pyyaml>=5.1.1
requests>=2.22.0