From 9e5c24e0825ecf3c88500f17f4a6a396a63127fc Mon Sep 17 00:00:00 2001 From: Morgan Jeon Date: Sun, 15 Oct 2023 16:26:34 +0900 Subject: [PATCH] Modified ftpsync --- scripts/config.yml | 33 +++++--- scripts/ftpsync.conf | 18 ++--- scripts/index.py | 42 +++++----- scripts/nginx/nginx.conf | 91 +++++++++++++++++++++ scripts/nginx/pawe.me.conf | 21 +++++ scripts/nginx/stub_status.conf | 11 +++ scripts/sync.sh | 21 +++-- scripts/templates/{ => default}/base.html | 0 scripts/templates/{ => default}/full.html | 0 scripts/templates/{ => default}/half.html | 0 scripts/templates/new/base.html | 99 +++++++++++++++++++++++ scripts/templates/new/full.html | 6 ++ scripts/templates/new/half.html | 6 ++ 13 files changed, 305 insertions(+), 43 deletions(-) create mode 100644 scripts/nginx/nginx.conf create mode 100644 scripts/nginx/pawe.me.conf create mode 100644 scripts/nginx/stub_status.conf rename scripts/templates/{ => default}/base.html (100%) rename scripts/templates/{ => default}/full.html (100%) rename scripts/templates/{ => default}/half.html (100%) create mode 100644 scripts/templates/new/base.html create mode 100644 scripts/templates/new/full.html create mode 100644 scripts/templates/new/half.html diff --git a/scripts/config.yml b/scripts/config.yml index f7d9b63..a2ce795 100644 --- a/scripts/config.yml +++ b/scripts/config.yml @@ -8,6 +8,8 @@ global: pre: post: log: logs/sync.log + log_dir: logs/ + data_dir: pub/ sync: - ALL @@ -18,8 +20,9 @@ defaults: fail: - scripts/alert.sh post: - - scripts/index.py + - scripts/index.py ${config} log: logs/${path}.log + delay: 4 rsync: options: "-rtlHpv --chmod=D0755,F0644 --partial --hard-links --safe-links --stats --delete --delete-after --delay-updates --max-delete=70000" exclude: @@ -35,47 +38,44 @@ defaults: repos: ARCHLINUX: url: rsync://mirrors.xtom.de/archlinux/ - delay: 4 name: "ArchLinux (x86_64)" path: archlinux UBUNTU: url: rsync://rsync.archive.ubuntu.com/ubuntu/ - delay: 4 name: "Ubuntu" path: ubuntu UBUNTU_CD: url: rsync://releases.ubuntu.com/releases/ - delay: 12 name: "Ubuntu Releases" + delay: 6 path: ubuntu_cd UBUNTU_CD_OLD: url: rsync://old-releases.ubuntu.com/releases/ - delay: 12 name: "Ubuntu Releases (Old)" + delay: 12 path: ubuntu_cd_old DEBIAN: - url: rsync://mirrors.xtom.jp/debian/ + url: rsync://mirrors.xtom.jp/ type: ftpsync - delay: 4 name: Debian path: debian scripts: + # pre: scripts/pre_debian.sh last_sync: "2023-10-12T10:58Z" DEBIAN_CD: url: rsync://ftp.lanet.kr/debian-cd/ - delay: 12 + delay: 6 name: "Debian CD" path: debian_cd FEDORA: url: rsync://dl.fedoraproject.org/fedora-enchilada/linux/ - delay: 4 name: "Fedora" path: fedora @@ -87,7 +87,6 @@ repos: MANJARO: url: rsync://ftp.riken.jp/manjaro/ - delay: 4 name: Manjaro path: manjaro @@ -105,6 +104,18 @@ repos: type: http delay: 12 + GNU: + name: "GNU" + path: gnu + url: rsync://ftp.gnu.org/gnu/ + delay: 12 + + LINUX: + name: "Linux Kernel" + path: linux + url: rsync://rsync.kernel.org/pub/ + delay: 12 + index: - ARCHLINUX - UBUNTU UBUNTU_CD @@ -115,4 +126,6 @@ index: - DIVIDER - ARCHLINUXARM - ASAHILINUX + - GNU + - LINUX - UBUNTU_CD_OLD \ No newline at end of file diff --git a/scripts/ftpsync.conf b/scripts/ftpsync.conf index 93ba9f4..7008b3f 100644 --- a/scripts/ftpsync.conf +++ b/scripts/ftpsync.conf @@ -1,12 +1,12 @@ -MIRRORNAME="pawe.me" -TO="${BASE_DIR}/debian" +MIRRORNAME="${DEB_HOST}" +TO="${DEB_PATH}" HUB=false -RSYNC_HOST="mirrors.xtom.jp" +RSYNC_HOST="${DEB_UPSTREAM}" RSYNC_PATH="debian" -INFO_MAINTAINER="Morgan " -INFO_COUNTRY=KR -INFO_LOCATION="Seoul, Korea" -INFO_THROUGHPUT=500Mb -LOGDIR="${BASE_DIR}/logs" +INFO_MAINTAINER="${DEB_ADMIN}" +INFO_COUNTRY="${DEB_COUNTRY}" +INFO_LOCATION="${DEB_LOCATION}" +INFO_THROUGHPUT=1Gbps +LOGDIR="${DEB_LOG}" UIPRETRIES=9 -TRACEHOST="PAWE.ME" \ No newline at end of file +TRACEHOST="${DEB_HOST}" \ No newline at end of file diff --git a/scripts/index.py b/scripts/index.py index 0e8daac..94d40e8 100755 --- a/scripts/index.py +++ b/scripts/index.py @@ -5,29 +5,35 @@ import yaml import jinja2 import datetime -BASE_DIR = "." -CONFIG_PATH = os.path.join(BASE_DIR, "scripts/config.yml") -TEMPLATES_DIR = os.path.join(BASE_DIR, "scripts/templates") -LOG_DIR = os.path.join(BASE_DIR, "logs") -SYNC_LOG = os.path.join(LOG_DIR, "sync.log") -OUTPUT_PATH = os.path.join(BASE_DIR, "index.html") - -def get_last_sync(repo_name): - if os.path.exists(SYNC_LOG): - with open(SYNC_LOG, 'r') as log_file: - log_all = reversed(log_file.readlines()) - for logline in log_all: - dist, time, stat = logline.split() - if repo_name == dist and stat =="SUCCESS": - return datetime.datetime.strptime(time, '%Y-%m-%dT%H:%MZ').strftime("%Y-%m-%d %H:%M") - return "Not Synced" +# def get_last_sync(repo_name): +# if os.path.exists(SYNC_LOG): +# with open(SYNC_LOG, 'r') as log_file: +# log_all = reversed(log_file.readlines()) +# for logline in log_all: +# dist, time, stat = logline.split() +# if repo_name == dist and stat =="SUCCESS": +# return datetime.datetime.strptime(time, '%Y-%m-%dT%H:%MZ').strftime("%Y-%m-%d %H:%M") +# return "Not Synced" # main() if __name__=="__main__": + + if len(sys,argv) == 3: + print(sys.argv) + + CONFIG_PATH = sys.argv[1] + with open(CONFIG_PATH, 'r') as f: config = yaml.safe_load(f) + + BASE_DIR = config.get("BASE_DIR", ".") + THEME = "new" + TEMPLATES_DIR = os.path.join(os.path.join(BASE_DIR, "scripts/templates"), THEME) + LOG_DIR = os.path.join(BASE_DIR, config['global'].get("log_dir", "logs")) + OUTPUT_PATH = os.path.join(os.path.join(BASE_DIR, config['global'].get("data_dir", ".")), "index.html") + env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATES_DIR)) base_template = env.get_template('base.html') full_template = env.get_template('full.html') @@ -50,8 +56,8 @@ if __name__=="__main__": context = { 'path': repo_data['path'], 'name': repo_data['name'], - 'lastsync': get_last_sync(repo_name), - # 'lastsync': repo_data.get('lastsync', "Not Synced"), + # 'lastsync': get_last_sync(repo_name), + 'lastsync': repo_data.get('lastsync', "Not Synced"), 'upstream': repo_data['url'] } print(context) diff --git a/scripts/nginx/nginx.conf b/scripts/nginx/nginx.conf new file mode 100644 index 0000000..2661e28 --- /dev/null +++ b/scripts/nginx/nginx.conf @@ -0,0 +1,91 @@ +user user; +worker_processes auto; +pid /run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; + +events { + worker_connections 768; + # multi_accept on; +} + +http { + + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + types_hash_max_size 2048; + # server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # SSL Settings + ## + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE + ssl_prefer_server_ciphers on; + + ## + # Logging Settings + ## + + log_format main_ext '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" ' + '"$host" sn="$server_name" ' 'rt=$request_time ' + 'ua="$upstream_addr" us="$upstream_status" ' + 'ut="$upstream_response_time" ul="$upstream_response_length" ' + 'cs=$upstream_cache_status' ; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Gzip Settings + ## + + gzip on; + + # gzip_vary on; + # gzip_proxied any; + # gzip_comp_level 6; + # gzip_buffers 16 8k; + # gzip_http_version 1.1; + # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + + ## + # Virtual Host Configs + ## + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} + + +#mail { +# # See sample authentication script at: +# # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript +# +# # auth_http localhost/auth.php; +# # pop3_capabilities "TOP" "USER"; +# # imap_capabilities "IMAP4rev1" "UIDPLUS"; +# +# server { +# listen localhost:110; +# protocol pop3; +# proxy on; +# } +# +# server { +# listen localhost:143; +# protocol imap; +# proxy on; +# } +#} diff --git a/scripts/nginx/pawe.me.conf b/scripts/nginx/pawe.me.conf new file mode 100644 index 0000000..a75fc04 --- /dev/null +++ b/scripts/nginx/pawe.me.conf @@ -0,0 +1,21 @@ +server { + listen 80 ; + listen [::]:80 ; + server_name repo.city; + + access_log /var/log/nginx/repo-city-access.log main_ext; + error_log /var/log/nginx/repo-city-error.log warn; + + root /srv/mirror; + + location /scripts/ { + return 404; + } + + location / { + autoindex on; + fancyindex on; + } +} + + diff --git a/scripts/nginx/stub_status.conf b/scripts/nginx/stub_status.conf new file mode 100644 index 0000000..f262466 --- /dev/null +++ b/scripts/nginx/stub_status.conf @@ -0,0 +1,11 @@ +server { + listen 127.0.0.1:80; + server_name 127.0.0.1; + location /nginx_status { + stub_status on; + access_log off; + allow 127.0.0.1; + deny all; + } +} + diff --git a/scripts/sync.sh b/scripts/sync.sh index c9f28da..b38073f 100755 --- a/scripts/sync.sh +++ b/scripts/sync.sh @@ -65,10 +65,10 @@ sync_repo() { return fi - path="${BASE_DIR}/$path/" + full_path="${BASE_DIR}/$(parse_yaml global.data_dir)/$path/" log="${BASE_DIR}/$log" - echo -e "--------\nRepo: $name\nType: $type\nUpstream: $url\nPath: $path\nLog: $log\n--------" + echo -e "--------\nRepo: $name\nType: $type\nUpstream: $url\nPath: $full_path\nLog: $log\n--------" if [[ ! -v DRY_RUN ]]; then @@ -82,18 +82,26 @@ sync_repo() { for ex in "${exclude_list[@]}"; do exclude="${exclude} --exclude='${ex}'" done - echo rsync ${rsync_options} ${exclude} $url $path >> $log - rsync ${rsync_options} ${exclude} $url $path >> $log 2>> ${log}-error + echo rsync ${rsync_options} ${exclude} $url $full_path >> $log + rsync ${rsync_options} ${exclude} $url $full_path >> $log 2>> ${log}-error ;; "ftpsync") cd ${BASE_DIR}/scripts - export BASE_DIR=${BASE_DIR} + + export DEB_ADMIN="Morgan " + export DEB_COUNTRY="Korea" + export DEB_LOCATION="Korea/Seoul" + export DEB_PATH="$full_path" + export DEB_HOST="$(parse_yaml global.hostname)" + export DEB_UPSTREAM="$url" + export DEB_LOG="${BASE_DIR}/$(parse_yaml global.log_dir)" + ./ftpsync >> $log 2>> ${log}-error cd ${BASE_DIR} ;; "http") echo ${repo} Fetch >> $log 2>> ${log}-error - python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error + python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $full_path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error echo ${repo} Download >> $log 2>> ${log}-error python3 -u $BASE_DIR/scripts/getFile.py $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error ;; @@ -277,6 +285,7 @@ for repo in "${repos[@]}"; do repo_post_scripts=($(get_repo_config ${repo} "scripts.post[]")) for script in "${repo_post_scripts[@]}"; do + script=${script/\${config\}/$CONFIG} execute $BASE_DIR/$script $repo done fi diff --git a/scripts/templates/base.html b/scripts/templates/default/base.html similarity index 100% rename from scripts/templates/base.html rename to scripts/templates/default/base.html diff --git a/scripts/templates/full.html b/scripts/templates/default/full.html similarity index 100% rename from scripts/templates/full.html rename to scripts/templates/default/full.html diff --git a/scripts/templates/half.html b/scripts/templates/default/half.html similarity index 100% rename from scripts/templates/half.html rename to scripts/templates/default/half.html diff --git a/scripts/templates/new/base.html b/scripts/templates/new/base.html new file mode 100644 index 0000000..2dc4b48 --- /dev/null +++ b/scripts/templates/new/base.html @@ -0,0 +1,99 @@ + + + + + + repo.city + + + + + + + + +
+

Welcome to REPO.CITY!

+

Simple mirroring & archiving server.

+
+
+
+

Mirroring List

+
+ + {{ repos }} + +
+ + +
+
+

Additional Mirror

+ +
+ + {{ repos_more }} + +
+

Image Files

+
+
+
+
+

Server Details

+

Location: Seoul, Korea

+

Bandwidth: ~1Gbps

+

Storage size: 32TiB

+

Sync delay: 4hrs

+

Contact: mirror@devpg.net

+

Maintainer: @morgan9e

+

This mirror is based on reposync.

+

For additional mirroring requests, please contact me at email above.

+
+ +
+ + + + + + \ No newline at end of file diff --git a/scripts/templates/new/full.html b/scripts/templates/new/full.html new file mode 100644 index 0000000..dc513f2 --- /dev/null +++ b/scripts/templates/new/full.html @@ -0,0 +1,6 @@ +
+

{{name}}

+

Last Updated: {{lastsync}}

+

Upstream: {{upstream}}

+
\ No newline at end of file diff --git a/scripts/templates/new/half.html b/scripts/templates/new/half.html new file mode 100644 index 0000000..548ecf6 --- /dev/null +++ b/scripts/templates/new/half.html @@ -0,0 +1,6 @@ +
+

{{name}}

+

Last Updated: {{lastsync}}

+

Upstream: {{upstream}}

+
\ No newline at end of file