diff --git a/index.html b/index.html
index 1e370c4..47802c7 100644
--- a/index.html
+++ b/index.html
@@ -40,13 +40,13 @@
-
Last Updated: Not Synced
+
Last Updated: 2023-10-12 11:24
Upstream: rsync://mirrors.xtom.de/archlinux/
-
Last Updated: Not Synced
+
Last Updated: 2023-10-12 11:24
Upstream: rsync://rsync.archive.ubuntu.com/ubuntu/
diff --git a/logs/old/.~tmp~ b/logs/old/.~tmp~
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/config.yml b/scripts/config.yml
index 82fa3e3..49c1bce 100644
--- a/scripts/config.yml
+++ b/scripts/config.yml
@@ -1,19 +1,25 @@
BASE_DIR: /srv/mirror
-ENV: ./scripts/.env
+
+# Any path under this must be in relative of BASE_DIR.
+# Script doesnt do any kind of outbound check, so make sure to check all paths.
+
global:
+ scripts:
+ pre:
+ post:
+ log: logs/sync.log
+ sync:
+ - ALL
+
+defaults:
type: rsync
scripts:
pre:
- # - ./scripts/cleanup.sh
fail:
- - ./scripts/alert.sh
+ - scripts/alert.sh
post:
- - ./script/index.py
- # - ./script/rotate_log.py
- log: ./logs/sync.log
- log_dir: ./logs/
- sync:
- - ALL
+ - scripts/index.py
+ log: logs/${path}.log
rsync:
options: "-rtlHpv --chmod=D0755,F0644 --partial --hard-links --safe-links --stats --delete --delete-after --delay-updates --max-delete=70000"
exclude:
@@ -25,35 +31,32 @@ global:
- "*.links.tar.gz*"
- "/other"
- "/sources"
+
repos:
ARCHLINUX:
url: rsync://mirrors.xtom.de/archlinux/
duration: 4
name: "ArchLinux (x86_64)"
path: archlinux
- log: ./logs/archlinux
- last_sync: "2023-10-09T23:48Z"
+
UBUNTU:
url: rsync://rsync.archive.ubuntu.com/ubuntu/
duration: 4
name: "Ubuntu"
path: ubuntu
- log: ./logs/ubuntu
- last_sync: "2023-10-09T23:48Z"
+
UBUNTU_CD:
url: rsync://releases.ubuntu.com/releases/
- duration: 24
+ duration: 12
name: "Ubuntu Releases"
path: ubuntu_cd
- log: ./logs/ubuntu_cd
- last_sync: "2023-10-09T23:48Z"
+
UBUNTU_CD_OLD:
url: rsync://old-releases.ubuntu.com/releases/
- duration: 24
+ duration: 12
name: "Ubuntu Releases (Old)"
path: ubuntu_cd_old
- log: ./logs/ubuntu_cd_old
- last_sync: "2023-10-09T23:48Z"
+
DEBIAN:
url: rsync://mirrors.xtom.jp/debian/
type: ftpsync
@@ -61,50 +64,47 @@ repos:
name: Debian
path: debian
scripts:
- # pre: ./scripts/pre_debian.sh
- last_sync: "2023-10-09T23:48Z"
+ # pre: scripts/pre_debian.sh
+ last_sync: "2023-10-12T10:58Z"
+
DEBIAN_CD:
url: rsync://ftp.lanet.kr/debian-cd/
- duration: 24
+ duration: 12
name: "Debian CD"
path: debian_cd
- log: ./logs/debian_cd
- last_sync: "2023-10-09T23:48Z"
+
FEDORA:
url: rsync://dl.fedoraproject.org/fedora-enchilada/linux/
duration: 4
name: "Fedora"
path: fedora
- log: ./logs/fedora
- last_sync: "2023-10-09T23:30Z"
+
RASPBIAN:
url: rsync://archive.raspbian.org/archive/
duration: 6
name: "Raspbian"
path: raspbian
- log: ./logs/raspbian
- last_sync: "2023-10-09T23:30Z"
+
MANJARO:
url: rsync://ftp.riken.jp/manjaro/
duration: 4
name: Manjaro
path: manjaro
- log: ./logs/manjaro
- last_sync: "2023-10-09T23:30Z"
+
ARCHLINUXARM:
name: "ArchLinux (ARM)"
path: archlinuxarm
url: http://jp.mirror.archlinuxarm.org/
type: http
duration: 12
- last_sync: "2023-10-09T23:30Z"
+
ASAHILINUX:
name: "AsahiLinux"
path: asahilinux
url: https://cdn.asahilinux.org/
type: http
duration: 12
- last_sync: "2023-10-09T23:30Z"
+
index:
- ARCHLINUX
- UBUNTU UBUNTU_CD
@@ -115,4 +115,4 @@ index:
- DIVIDER
- ARCHLINUXARM
- ASAHILINUX
- - UBUNTU_CD_OLD
+ - UBUNTU_CD_OLD
\ No newline at end of file
diff --git a/scripts/index.py b/scripts/index.py
index 58de335..0e8daac 100755
--- a/scripts/index.py
+++ b/scripts/index.py
@@ -6,22 +6,21 @@ import jinja2
import datetime
BASE_DIR = "."
-
CONFIG_PATH = os.path.join(BASE_DIR, "scripts/config.yml")
TEMPLATES_DIR = os.path.join(BASE_DIR, "scripts/templates")
LOG_DIR = os.path.join(BASE_DIR, "logs")
+SYNC_LOG = os.path.join(LOG_DIR, "sync.log")
OUTPUT_PATH = os.path.join(BASE_DIR, "index.html")
-# def get_last_sync(repo_name):
-# log_path = os.path.join(LOG_DIR, f"all.log")
-# if os.path.exists(log_path):
-# with open(log_path, 'r') as log_file:
-# log_all = reversed(log_file.readlines())
-# for logline in log_all:
-# time, stat, dist = logline.split()
-# if repo_name == dist and stat =="DONE":
-# return datetime.datetime.strptime(time, '%Y%m%d_%H%M').strftime("%Y-%m-%d %H:%M")
-# return "Not Synced"
+def get_last_sync(repo_name):
+ if os.path.exists(SYNC_LOG):
+ with open(SYNC_LOG, 'r') as log_file:
+ log_all = reversed(log_file.readlines())
+ for logline in log_all:
+ dist, time, stat = logline.split()
+ if repo_name == dist and stat =="SUCCESS":
+ return datetime.datetime.strptime(time, '%Y-%m-%dT%H:%MZ').strftime("%Y-%m-%d %H:%M")
+ return "Not Synced"
# main()
@@ -51,7 +50,8 @@ if __name__=="__main__":
context = {
'path': repo_data['path'],
'name': repo_data['name'],
- 'lastsync': repo_data.get('lastsync', "Not Synced"),
+ 'lastsync': get_last_sync(repo_name),
+ # 'lastsync': repo_data.get('lastsync', "Not Synced"),
'upstream': repo_data['url']
}
print(context)
diff --git a/scripts/sync.sh b/scripts/sync.sh
index 9599a23..2ab227c 100755
--- a/scripts/sync.sh
+++ b/scripts/sync.sh
@@ -1,66 +1,109 @@
#!/bin/bash
-CONFIG_PATH="./scripts/config.yml"
+# Default locations.
+CONFIG="./scripts/config.yml"
YAML="./scripts/yq"
-BASE_DIR=$($YAML eval ".BASE_DIR" "${CONFIG_PATH}")
-TIMESTAMP=$(date '+%Y-%m-%dT%H:%MZ')
+function usage() {
+ cat <v4.35.2)
+ -v, --verbose
+ -> -b, --base-dir: overrides base_dir
+ -> -d, --dry-run: dry run
+EOF
+ exit 1
+}
+
+function error() {
+ echo $0: "$1"
+ exit
+}
+
+function debug() {
+ if [[ -v DEBUG ]]; then
+ echo debug: "$1"
+ fi
+}
+
+function execute() {
+ debug "Executing script: \"$@\""
+ if [[ ! -v DRY_RUN ]]; then
+ $@
+ fi
+}
parse_yaml() {
- echo $($YAML eval ".$1" "${CONFIG_PATH}")
+ echo $($YAML eval ".$1" "${CONFIG}")
+}
+
+get_repo_config() {
+ local LOCAL_CFG=$($YAML eval ".repos.$1.$2" "${CONFIG}")
+ if [[ "$LOCAL_CFG" == "null" || "$LOCAL_CFG" == "" ]]; then
+ echo $($YAML eval ".defaults.$2" "${CONFIG}");
+ return 1
+ fi
+ echo $LOCAL_CFG
+ return 0
+
+ # 1 if global 0 if local
}
sync_repo() {
-
local repo=$1
- local type=$(parse_yaml "repos.${repo}.type")
- local url=$(parse_yaml "repos.${repo}.url")
- local path=$(parse_yaml "repos.${repo}.path")
- local log=$(parse_yaml "repos.${repo}.log")
+ local name=$(get_repo_config ${repo} name)
+ local type=$(get_repo_config ${repo} type)
+ local url=$(get_repo_config ${repo} url)
+ local path=$(get_repo_config ${repo} path)
+ local log; log=$(get_repo_config ${repo} log) || log=${log/\${path\}/$path};
- if [[ "$type" == "null" ]]; then
- type=$(parse_yaml "global.type")
+ if [[ "$type" == "null" || "$url" == "null" || "$path" == "null" || "$log" == "null" ]]; then
+ echo "Error config wrong."
+ return
fi
- if [[ "$log" == "null" ]]; then
- log="$(parse_yaml "global.log_dir")$path"
- fi
-
- log=$(realpath $log)
path="${BASE_DIR}/$path/"
- echo -e "----\n| Repo: $repo\n| Type: $type\n| Upstream: $url\n| Path: $path\n| Log: $log\n----"
-
- rotate_log $log
+ log="${BASE_DIR}/$log"
- case $type in
- "rsync")
- local rsync_options=$(parse_yaml 'global.rsync.options')
- local exclude_list=($(parse_yaml 'global.rsync.exclude[]'))
- local exclude=""
- for ex in "${exclude_list[@]}"; do
- exclude="${exclude} --exclude='${ex}'"
- done
- echo rsync ${rsync_options} ${exclude} $url $path >> $log
- rsync ${rsync_options} ${exclude} $url $path >> $log 2>> ${log}-error
- ;;
- "ftpsync")
- cd ${BASE_DIR}/scripts
- export BASE_DIR=${BASE_DIR}
- ./ftpsync >> $log 2>> ${log}-error
- cd ${BASE_DIR}
- ;;
- "http")
- echo ${repo} Fetch >> $log 2>> ${log}-error
- python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
- echo ${repo} Download >> $log 2>> ${log}-error
- python3 -u $BASE_DIR/scripts/getFile.py $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
- ;;
- *)
- echo "Unknown type $type for $repo." | tee ${log}-error
- ;;
- esac
+ echo -e "--------\nRepo: $name\nType: $type\nUpstream: $url\nPath: $path\nLog: $log\n--------"
- clean_log $log
+ if [[ ! -v DRY_RUN ]]; then
+
+ rotate_log $log
+
+ case $type in
+ "rsync")
+ local rsync_options=$(get_repo_config $repo 'rsync.options')
+ local exclude_list=($(get_repo_config $repo 'rsync.exclude[]'))
+ local exclude=""
+ for ex in "${exclude_list[@]}"; do
+ exclude="${exclude} --exclude='${ex}'"
+ done
+ echo rsync ${rsync_options} ${exclude} $url $path >> $log
+ rsync ${rsync_options} ${exclude} $url $path >> $log 2>> ${log}-error
+ ;;
+ "ftpsync")
+ cd ${BASE_DIR}/scripts
+ export BASE_DIR=${BASE_DIR}
+ ./ftpsync >> $log 2>> ${log}-error
+ cd ${BASE_DIR}
+ ;;
+ "http")
+ echo ${repo} Fetch >> $log 2>> ${log}-error
+ python3 -u $BASE_DIR/scripts/getFetch.py "${url}" $path $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
+ echo ${repo} Download >> $log 2>> ${log}-error
+ python3 -u $BASE_DIR/scripts/getFile.py $BASE_DIR/scripts/${path}.fetch >> $log 2>> ${log}-error
+ ;;
+ *)
+ echo "Unknown type $type for $repo." | tee ${log}-error
+ ;;
+ esac
+
+ clean_log $log
+ fi
}
rotate_log() {
@@ -92,50 +135,158 @@ clean_log() {
fi
}
+function global_log() {
+ GLOBAL_LOG_FILE="$(parse_yaml 'global.log')"
+ echo "$1 $TIMESTAMP SUCCESS" >> $GLOBAL_LOG_FILE
+}
+
+function global_log_error() {
+ GLOBAL_LOG_FILE="$(parse_yaml 'global.log')"
+ echo "$1 $TIMESTAMP ERROR" >> $GLOBAL_LOG_FILE
+}
+
##########
# Main #
##########
-echo Started job $TIMESTAMP..
-
-global_pre_scripts=($(parse_yaml 'global.scripts.pre[]'))
-for script in "${global_pre_scripts[@]}"; do
- $script
+while [ "$1" != "" ]; do
+ case $1 in
+ -c | --config)
+ shift
+ if [[ -z $1 ]]; then
+ error "option '-c' requires argument 'config_path'"
+ fi
+ CONFIG=$1
+ ;;
+ -y | --yq)
+ shift
+ if [[ -z $1 ]]; then
+ error "option '-y' requires argument 'yq_path'"
+ fi
+ YAML=$1
+ ;;
+ -b | --base-dir)
+ shift
+ if [[ -z $1 ]]; then
+ error "option '-b' requires argument 'base_dir'"
+ fi
+ BASE_DIR_OVERRIDE=$1
+ ;;
+ -h | --help)
+ usage
+ ;;
+ -v | --verbose)
+ DEBUG=1
+ ;;
+ -d | --dry-run)
+ DRY_RUN=1
+ ;;
+ *)
+ usage
+ exit 1
+ ;;
+ esac
+ shift
done
+if [[ -v DEBUG ]]; then
+ debug "DEBUG=1"
+fi
+if [[ -v DRY_RUN ]]; then
+ debug "DRY_RUN=1"
+fi
+debug CONFIG="\"${CONFIG}\""
+debug YQ="\"${YAML}\""
+
+if [[ ! -f ${CONFIG} ]]; then
+ error "config not found."
+fi
+
+if [[ ! -f ${YAML} ]]; then
+ error "yq not found."
+fi
+
+BASE_DIR=$($YAML eval ".BASE_DIR" "${CONFIG}")
+
+if [[ -v DEBUG && -v BASE_DIR_OVERRIDE ]]; then
+ debug "Overriding $BASE_DIR to $BASE_DIR_OVERRIDE"
+ BASE_DIR="$BASE_DIR_OVERRIDE"
+fi
+
+TIMESTAMP=$(date '+%Y-%m-%dT%H:%MZ')
+
+debug BASE_DIR="\"${BASE_DIR}\""
+debug TIMESTAMP="\"${TIMESTAMP}\""
+
+echo Started job $TIMESTAMP..
+
+cd $BASE_DIR
+# PRE
+global_pre_scripts=($(parse_yaml 'global.scripts.pre[]'))
+for script in "${global_pre_scripts[@]}"; do
+ execute $BASE_DIR/$script
+done
+#
repos=($(parse_yaml 'global.sync[]'))
if [[ "${repos[0]}" == "ALL" ]]; then
- repos=($($YAML eval '.repos | keys| .[]' "${CONFIG_PATH}"))
+ repos=($($YAML eval '.repos | keys | .[]' "${CONFIG}"))
fi
for repo in "${repos[@]}"; do
- echo Checking $repo...
- duration=$(parse_yaml "repos.${repo}.duration")
- last_sync_timestamp=$(date -d "$(parse_yaml "repos.${repo}.last_sync")" +%s)
+ cd $BASE_DIR
+ debug "Checking $repo..."
+
+ duration=$(get_repo_config ${repo} "duration")
+ last_sync_timestamp=$(date -d "$(get_repo_config ${repo} "last_sync")" +%s)
next_sync_timestamp=$(( last_sync_timestamp + duration * 3600 ))
- next_sync_timestamp=1
+
+ if [[ -v DEBUG ]]; then
+ next_sync_timestamp=1
+ # read -p "Continue? " choice
+ # case "$choice" in
+ # y) next_sync_timestamp=1;;
+ # *) continue;;
+ # esac
+ fi
+
if [ $next_sync_timestamp -le $(date +%s) ]; then
- echo "Lastsync $last_sync_timestamp"
+ debug "Lastsync was $last_sync_timestamp."
echo "Syncing $repo..."
+ repo_pre_scripts=($(get_repo_config ${repo} "scripts.pre[]"))
+ for script in "${repo_pre_scripts[@]}"; do
+ execute $BASE_DIR/$script $repo
+ done
+
sync_repo $repo
if [ $? -ne 0 ]; then
- global_fail_scripts=($(parse_yaml 'global.scripts.fail[]'))
- for script in "${global_fail_scripts[@]}"; do
- $script
+ repo_fail_scripts=($(get_repo_config ${repo} "scripts.fail[]"))
+ for script in "${repo_fail_scripts[@]}"; do
+ execute $BASE_DIR/$script $repo
done
- echo "Error syncing $repo"
+
+ global_log_error $repo
+ echo "Error during syncing $repo."
else
- $YAML eval ".repos.${repo}.last_sync = \"$TIMESTAMP\"" -i "${CONFIG_PATH}"
+
+ global_log $repo
+ $YAML eval ".repos.${repo}.last_sync = \"$TIMESTAMP\"" -i "${CONFIG}"
echo "Successfully synced $repo."
fi
+
+ repo_post_scripts=($(get_repo_config ${repo} "scripts.post[]"))
+ for script in "${repo_post_scripts[@]}"; do
+ execute $BASE_DIR/$script $repo
+ done
fi
done
+# POST
global_post_scripts=($(parse_yaml 'global.scripts.post[]'))
for script in "${global_post_scripts[@]}"; do
- $script
+ execute $BASE_DIR/$script
done
+#
-echo Ended job $TIMESTAMP..
+echo Ended job $TIMESTAMP..
\ No newline at end of file