Sadiq's Knowledge Vaults

Backup to AWS

3 min read

  1. sudo apt update && sudo apt install -y unzip && curl “https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip” -o “/tmp/awscliv2.zip” && cd /tmp && unzip awscliv2.zip && sudo ./aws/install && aws —version
  2. aws —version
  3. aws configure
    • if this command not work, create manually, nano /home/dawg1/.aws/credentials
[default]
aws_access_key_id = AKIA2UC3EGF3PWPBWZSH
aws_secret_access_key = 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
  1. cd /usr/local/bin
  2. sudo nano aws_upload_video.sh
    sudo chmod +x aws_upload_video.sh
  3. check log file and usernames in paths
#!/usr/bin/env bash
set -euo pipefail
 
# --- Configuration ---
VIDEO_DIR="/home/dawg1/Downloads/raspberry_pie_camera_capture/recordings"
S3_BUCKET="s3://raspberry-pie-camera-capture"
PORT="port4"
LOG_FILE="/home/dawg1/upload_videos.log"
MAX_LOG_SIZE=10485760
AWS_CREDENTIALS_FILE="$HOME/.aws/credentials"
AWS_CONFIG_FILE="$HOME/.aws/config"
 
# Extract AWS_PROFILE
AWS_PROFILE=$(grep '^\[' "$AWS_CREDENTIALS_FILE" | head -n1 | sed 's/^\[\(.*\)\]/\1/')
 
# --- Credentials & Region ---
export AWS_SHARED_CREDENTIALS_FILE="$AWS_CREDENTIALS_FILE"
export AWS_CONFIG_FILE="$AWS_CONFIG_FILE"
export AWS_PROFILE="$AWS_PROFILE"
export AWS_DEFAULT_REGION="ap-south-1"
 
# --- Utility Functions ---
log_message() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
 
rotate_log() {
    if [[ -f "$LOG_FILE" ]] && (( $(stat -c%s "$LOG_FILE") > MAX_LOG_SIZE )); then
        mv "$LOG_FILE" "${LOG_FILE}.old"
        log_message "Log file rotated"
    fi
}
 
check_aws_auth() {
    if ! command -v aws &>/dev/null; then
        log_message "ERROR: aws CLI not found"
        exit 1
    fi
}
 
get_local_size() {
    stat -c%s "$1"
}
 
get_remote_size() {
    aws s3api head-object \
        --bucket "${S3_BUCKET#s3://}" \
        --key "$1" \
        --query 'ContentLength' \
        --output text 2>/dev/null || echo 0
}
 
upload_and_delete() {
    local file_path="$1"
    local rel="$2"
    local filename=$(basename "$file_path")
    local s3_key="$PORT/$rel"
 
    log_message "DEBUG: Uploading $file_path to $S3_BUCKET/$s3_key"
 
    if aws s3api head-object --bucket "${S3_BUCKET#s3://}" --key "$s3_key" &>/dev/null; then
        local remote_size
        remote_size=$(get_remote_size "$s3_key")
        local local_size
        local_size=$(get_local_size "$file_path")
 
        local diff=$(( remote_size > local_size ? remote_size - local_size : local_size - remote_size ))
        if (( diff < 1048576 )); then
            log_message "Exists on S3 with size close to local: $filename (local $local_size bytes, remote $remote_size bytes)."
            return
        fi
    fi
 
    if aws s3 cp "$file_path" "$S3_BUCKET/$s3_key"; then
        local remote_size=$(get_remote_size "$s3_key")
        local local_size=$(get_local_size "$file_path")
        local diff=$(( remote_size > local_size ? remote_size - local_size : local_size - remote_size ))
 
        if (( diff < 1048576 )); then
            log_message " -> Upload successful with acceptable size difference ($local_size vs $remote_size)."
            # rm "$file_path"
        else
            log_message " -> ERROR: Size mismatch after upload (local $local_size vs remote $remote_size)."
        fi
    else
        log_message " -> ERROR: aws s3 cp failed."
    fi
}
 
# --- Main ---
main() {
    rotate_log
    log_message "====== Starting Video Upload ======"
    log_message "Using AWS Profile: $AWS_PROFILE"
 
    [[ -d "$VIDEO_DIR" ]] || { log_message "ERROR: VIDEO_DIR not found"; exit 1; }
    [[ -w "$VIDEO_DIR" ]] || { log_message "ERROR: VIDEO_DIR not writable"; exit 1; }
 
    check_aws_auth
 
    # Find relative file list
    mapfile -t files < <(
        cd "$VIDEO_DIR" && \
        find . -type f \( -iname "*.mp4" -o -iname "*.mkv" -o -iname "*.csv" \) -printf "%P\n"
    )
 
    if (( ${#files[@]} == 0 )); then
        log_message "No files found"
        return
    fi
 
    log_message "Found ${#files[@]} files. Processing..."
 
    for rel in "${files[@]}"; do
        upload_and_delete "$VIDEO_DIR/$rel" "$rel"
    done
 
    log_message "====== Upload Process Finished ======"
}
 
main "$@"
  1. cd /etc/systemd/system
    sudo nano aws.upload.service
  2. change user in this
[Unit]
Description=My Custom Shell Script Service to Trigger aws upload
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/bin/aws_upload_video.sh
Restart=on-failure
User=dawg10
WorkingDirectory=/usr/local/bin
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

[Install]
WantedBy=multi-user.target

  1. sudo systemctl start camera.service
    sudo systemctl stop camera.service

    sudo systemctl start aws.upload.service
    sudo systemctl stop aws.upload.service

  2. Command to ls the objects replace <bucket_name>

aws s3api list-objects --bucket <bucket_name> --query "Contents[].Key" --output text | tr '\t' '\n' | sort | awk -F'/' '
{
  path = ""
  for (i=1; i<=NF; i++) {
    path = (path ? path "/" : "") $i
    if (!(path in seen)) {
      seen[path] = 1
      level[i] = $i
      indent = ""
      for (j=1; j<i; j++) indent = indent "│   "
      if (i == NF)
        printf "%s└── %s\n", indent, $i
      else
        printf "%s├── %s\n", indent, $i
    }
  }
}'

Graph View