On Wed, Mar 4, 2020 at 6:04 PM Kevin Fenzi kevin@scrye.com wrote:
We have been getting complaints from copr users that they are hitting out of date cloudfront cached data when they are doing builds. We are syncing not all that often currently, and sometimes if a updates push or rawhide compose finishes after the sync time it coud be a long while before it picks up on it. So, since most of these jobs finish in 5-10min when there is nothing to sync, just have them all run every 15min or so. If this starts hitting locking too much we can spread them back out once we get a sense of when they are hitting that.
Additionally, we should just set them up to only sync when their particular thing has finished. This would make it a lot more sane, but require a redesign/rewrite.
Signed-off-by: Kevin Fenzi kevin@scrye.com
roles/s3-mirror/tasks/main.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/roles/s3-mirror/tasks/main.yml b/roles/s3-mirror/tasks/main.yml index 12351cb..6916d01 100644 --- a/roles/s3-mirror/tasks/main.yml +++ b/roles/s3-mirror/tasks/main.yml @@ -60,7 +60,7 @@
s3-mirror
name: s3sync cron - test releases
cron: name="s3sync-updates-current" minute="40" hour="5,9,13,19" user="s3-mirror"
- cron: name="s3sync-updates-current" minute="1,16,31,46" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-test-releases "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/releases/test/" 2>&1 | /usr/local/bin/nag-once s3-test-releases.sh 1d 2>&1' cron_file=s3-test-releases.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
@@ -68,7 +68,7 @@
- s3-mirror
I dont think this change is needed, we only sync Beta content there, why run the cron every 15 min.
- name: s3sync cron - updates for current
- cron: name="s3sync-updates-current" minute="0" hour="3,9,15,21" user="s3-mirror"
- cron: name="s3sync-updates-current" minute="2,17,32,47" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-updates-current "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int }}/" 2>&1 | /usr/local/bin/nag-once s3-updates-current.sh 1d 2>&1' cron_file=s3-updates-current.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
@@ -76,7 +76,7 @@
s3-mirror
name: s3sync cron - updates for development/current+1 x86_64
cron: name="s3sync-updates-current" minute="0" hour="2,7,10" user="s3-mirror"
- cron: name="s3sync-updates-current" minute="3,18,33,48" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-updates-dev-cur-plus-1-x86_64 "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/development/{{ FedoraCycleNumber|int + 1 }}/Everything/x86_64/os/" 2>&1 | /usr/local/bin/nag-once s3-updates-dev-cur-plus-1-x86_64.sh 1d 2>&1' cron_file=s3-updates-dev-cur-plus-1-x86_64.sh disabled={{not FedoraBranched|bool}}
@@ -85,7 +85,7 @@
s3-mirror
name: s3sync cron - updates for development/current+1 aarch64
cron: name="s3sync-updates-current" minute="0" hour="4,11,18" user="s3-mirror"
- cron: name="s3sync-updates-current" minute="4,19,34,49" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-updates-dev-cur-plus-1-aarch64 "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/development/{{ FedoraCycleNumber|int + 1 }}/Everything/aarch64/os/" 2>&1 | /usr/local/bin/nag-once s3-updates-dev-cur-plus-1-aarch64.sh 1d 2>&1' cron_file=s3-updates-dev-cur-plus-1-aarch64.sh disabled={{not FedoraBranched|bool}}
@@ -94,7 +94,7 @@
s3-mirror
name: s3sync cron - updates for current-1
cron: name="s3sync-updates-previous" minute="30" hour="0,6,12,18" user="s3-mirror"
- cron: name="s3sync-updates-previous" minute="5,20,35,50" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-updates-previous "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int - 1 }}/" 2>&1 | /usr/local/bin/nag-once s3-updates-previous.sh 1d 2>&1' cron_file=s3-updates-previous.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
@@ -102,7 +102,7 @@
s3-mirror
name: s3sync cron - epel 7 x86_64
cron: name="s3sync-epel7-x86_64" minute="10" hour="2,5,8,11,14,17,20,23" user="s3-mirror"
- cron: name="s3sync-epel7-x86_64" minute="6,21,36,51" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel7-x86_64 "/usr/local/bin/s3-sync-path.sh /pub/epel/7/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-epel7-x86_64.sh 1d 2>&1' cron_file=s3-epel7-x86_64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
@@ -110,7 +110,7 @@
s3-mirror
name: s3sync cron - epel 7 aarch64
cron: name="s3sync-epel7-aarch64" minute="20" hour="4,7,10,13,16,19,22" user="s3-mirror"
- cron: name="s3sync-epel7-aarch64" minute="7,22,37,52" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel7-aarch64 "/usr/local/bin/s3-sync-path.sh /pub/epel/7/aarch64/" 2>&1 | /usr/local/bin/nag-once s3-epel7-aarch64.sh 1d 2>&1' cron_file=s3-epel7-aarch64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
@@ -118,7 +118,7 @@
s3-mirror
name: s3sync cron - epel 8 Everything x86_64
cron: name="s3sync-epel8-everything-x86_64" minute="43" hour="3,6,9,12,15,17,20,23" user="s3-mirror"
- cron: name="s3sync-epel8-everything-x86_64" minute="8,23,38,53" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel8-everything-x86_64 "/usr/local/bin/s3-sync-path.sh /pub/epel/8/Everything/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-epel8-everything-x86_64.sh 1d 2>&1' cron_file=s3-epel8-everything-x86_64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
@@ -126,7 +126,7 @@
s3-mirror
name: s3sync cron - epel 8 Everything aarch64
cron: name="s3sync-epel8-everything-aarch64" minute="38" hour="4,7,10,13,16,19,22" user="s3-mirror"
- cron: name="s3sync-epel8-everything-aarch64" minute="9,24,39,54" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel8-everything-aarch64 "/usr/local/bin/s3-sync-path.sh /pub/epel/8/Everything/aarch64/" 2>&1 | /usr/local/bin/nag-once s3-epel8-everything-aarch64.sh 1d 2>&1' cron_file=s3-epel8-everything-aarch64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
@@ -134,7 +134,7 @@
s3-mirror
name: s3sync cron - epel 8 Modular x86_64
cron: name="s3sync-epel8-modular-x86_64" minute="32" hour="3,6,9,12,15,17,20,23" user="s3-mirror"
- cron: name="s3sync-epel8-modular-x86_64" minute="10,25,40,55" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel8-modular-x86_64 "/usr/local/bin/s3-sync-path.sh /pub/epel/8/Modular/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-epel8-modular-x86_64.sh 1d 2>&1' cron_file=s3-epel8-modular-x86_64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
@@ -142,7 +142,7 @@
s3-mirror
name: s3sync cron - epel 8 Modular aarch64
cron: name="s3sync-epel8-modular-aarch64" minute="27" hour="4,7,10,13,16,19,22" user="s3-mirror"
- cron: name="s3sync-epel8-modular-aarch64" minute="11,26,41,56" user="s3-mirror" job='/usr/local/bin/lock-wrapper s3sync-epel8-modular-aarch64 "/usr/local/bin/s3-sync-path.sh /pub/epel/8/Modular/aarch64/" 2>&1 | /usr/local/bin/nag-once s3-epel8-modular-aarch64.sh 1d 2>&1' cron_file=s3-epel8-modular-aarch64.sh when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
-- 1.8.3.1 _______________________________________________ infrastructure mailing list -- infrastructure@lists.fedoraproject.org To unsubscribe send an email to infrastructure-leave@lists.fedoraproject.org Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/ List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines List Archives: https://lists.fedoraproject.org/archives/list/infrastructure@lists.fedorapro...
Overall, +1