From 5ff58867f40a8c4e3cd82be8f8672a59d8aa1fdd Mon Sep 17 00:00:00 2001 From: Neha Prasad Date: Fri, 7 Jul 2017 12:37:17 -0400 Subject: [PATCH 1/5] Koji perm_id cache --- .../builder/KojiModuleBuilder.py | 9 ++++++- module_build_service/builder/koji_cache.py | 22 +++++++++++++++++ .../builder/setup_memcache.py | 24 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 module_build_service/builder/koji_cache.py create mode 100644 module_build_service/builder/setup_memcache.py diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index eede0abe..b691c7a0 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -30,6 +30,7 @@ import tempfile import glob import datetime import time +import dogpile.cache import random import string import kobo.rpmlib @@ -56,6 +57,7 @@ class KojiModuleBuilder(GenericBuilder): backend = "koji" _build_lock = threading.Lock() + newRegion = make_region().configure("dogpile.cache.memory") @module_build_service.utils.validate_koji_tag('tag_name') def __init__(self, owner, module, config, tag_name, components): @@ -90,6 +92,10 @@ class KojiModuleBuilder(GenericBuilder): return "" % ( self.module_str, self.tag_name) + @dogpile.cache.memory.cache_on_arguments() + def getPerms(self): + return dict([(p['name'], p['id']) for p in self.koji_session.getAllPerms()]) + @module_build_service.utils.retry(wait_on=(IOError, koji.GenericError)) def buildroot_ready(self, artifacts=None): """ @@ -598,7 +604,8 @@ chmod 644 %buildroot/%_rpmconfigdir/macros.d/macros.modules if taginfo['locked']: raise SystemError("Tag %s: master lock already set. Can't edit tag" % taginfo['name']) - perm_ids = dict([(p['name'], p['id']) for p in self.koji_session.getAllPerms()]) + perm_ids = self.getPerms() + if perm not in perm_ids: raise ValueError("Unknown permissions %s" % perm) diff --git a/module_build_service/builder/koji_cache.py b/module_build_service/builder/koji_cache.py new file mode 100644 index 00000000..71714fe4 --- /dev/null +++ b/module_build_service/builder/koji_cache.py @@ -0,0 +1,22 @@ +import dogpile.cache +from KojiModuleBuilder import KojiModuleBuilder +import pylibmc +import koji + +#set up CacheRegion +koji_perm_ids = pylibmc.Client(["127.0.0.1"], binary=True, + behaviors={"tcp_nodelay": True, + "ketama": True}) + +region = make_region().configure( + 'koji_perm_ids.pylibmc', + arguments = { + 'url': ["127.0.0.1"], + } +) + +@region.cache_on_arguments() +def generate_perm_id(perm): + + + diff --git a/module_build_service/builder/setup_memcache.py b/module_build_service/builder/setup_memcache.py new file mode 100644 index 00000000..d438a389 --- /dev/null +++ b/module_build_service/builder/setup_memcache.py @@ -0,0 +1,24 @@ +/*set up memcache server for koji perm_ids*/ + +my $memclient = Cache::Memcached->new({ servers => [ '10.0.0.10:11211', '10.0.0.11:11211' ]}); +memcli = new Memcache +memcli:add_server('10.0.0.10:11211') + +sql = "SELECT * FROM user WHERE user_id = ?" +key = 'SQL:' . user_id . ':' . md5sum(sql) + +if (defined result = memcli:get(key)) { + return result +} else { + handler = run_sql(sql, user_id) + + # Often what you get back when executing SQL is a special handler + # object. You can't directly cache this. Stick to strings, arrays, + # and hashes/dictionaries/tables + rows_array = handler:turn_into_an_array + + # Cache it for five minutes + memcli:set(key, rows_array, 5 * 60) + + return rows_array +} From a05b485c432e124e92182d47cec734f70c8e903e Mon Sep 17 00:00:00 2001 From: Neha Prasad Date: Fri, 7 Jul 2017 12:37:48 -0400 Subject: [PATCH 2/5] Koji perm_id cache --- module_build_service/builder/koji_cache.py | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 module_build_service/builder/koji_cache.py diff --git a/module_build_service/builder/koji_cache.py b/module_build_service/builder/koji_cache.py deleted file mode 100644 index 71714fe4..00000000 --- a/module_build_service/builder/koji_cache.py +++ /dev/null @@ -1,22 +0,0 @@ -import dogpile.cache -from KojiModuleBuilder import KojiModuleBuilder -import pylibmc -import koji - -#set up CacheRegion -koji_perm_ids = pylibmc.Client(["127.0.0.1"], binary=True, - behaviors={"tcp_nodelay": True, - "ketama": True}) - -region = make_region().configure( - 'koji_perm_ids.pylibmc', - arguments = { - 'url': ["127.0.0.1"], - } -) - -@region.cache_on_arguments() -def generate_perm_id(perm): - - - From df47d3e28c81a1baf1eeb939e5358e829a2b94a0 Mon Sep 17 00:00:00 2001 From: Neha Prasad Date: Fri, 14 Jul 2017 11:44:26 -0400 Subject: [PATCH 3/5] delete setup_memcache.py --- .../builder/setup_memcache.py | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 module_build_service/builder/setup_memcache.py diff --git a/module_build_service/builder/setup_memcache.py b/module_build_service/builder/setup_memcache.py deleted file mode 100644 index d438a389..00000000 --- a/module_build_service/builder/setup_memcache.py +++ /dev/null @@ -1,24 +0,0 @@ -/*set up memcache server for koji perm_ids*/ - -my $memclient = Cache::Memcached->new({ servers => [ '10.0.0.10:11211', '10.0.0.11:11211' ]}); -memcli = new Memcache -memcli:add_server('10.0.0.10:11211') - -sql = "SELECT * FROM user WHERE user_id = ?" -key = 'SQL:' . user_id . ':' . md5sum(sql) - -if (defined result = memcli:get(key)) { - return result -} else { - handler = run_sql(sql, user_id) - - # Often what you get back when executing SQL is a special handler - # object. You can't directly cache this. Stick to strings, arrays, - # and hashes/dictionaries/tables - rows_array = handler:turn_into_an_array - - # Cache it for five minutes - memcli:set(key, rows_array, 5 * 60) - - return rows_array -} From 83b0c1562e39c9f9fe2dc644f9a3798b741651ef Mon Sep 17 00:00:00 2001 From: Neha Prasad Date: Fri, 14 Jul 2017 13:58:16 -0400 Subject: [PATCH 4/5] fix koji cache --- module_build_service/builder/KojiModuleBuilder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index b691c7a0..989f034e 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -57,7 +57,7 @@ class KojiModuleBuilder(GenericBuilder): backend = "koji" _build_lock = threading.Lock() - newRegion = make_region().configure("dogpile.cache.memory") + region = dogpile.cache.make_region() @module_build_service.utils.validate_koji_tag('tag_name') def __init__(self, owner, module, config, tag_name, components): @@ -92,7 +92,7 @@ class KojiModuleBuilder(GenericBuilder): return "" % ( self.module_str, self.tag_name) - @dogpile.cache.memory.cache_on_arguments() + @region.cache_on_arguments() def getPerms(self): return dict([(p['name'], p['id']) for p in self.koji_session.getAllPerms()]) From 459ffb241d15192590e490c7b819eedde64742ac Mon Sep 17 00:00:00 2001 From: Neha Prasad Date: Mon, 17 Jul 2017 08:56:17 -0400 Subject: [PATCH 5/5] Configure dogpile.cache --- module_build_service/builder/KojiModuleBuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_build_service/builder/KojiModuleBuilder.py b/module_build_service/builder/KojiModuleBuilder.py index 989f034e..d1832f7a 100644 --- a/module_build_service/builder/KojiModuleBuilder.py +++ b/module_build_service/builder/KojiModuleBuilder.py @@ -57,7 +57,7 @@ class KojiModuleBuilder(GenericBuilder): backend = "koji" _build_lock = threading.Lock() - region = dogpile.cache.make_region() + region = dogpile.cache.make_region().configure('dogpile.cache.memory') @module_build_service.utils.validate_koji_tag('tag_name') def __init__(self, owner, module, config, tag_name, components):