scripts/import-repo.shblame
View source
3e3af551#!/usr/bin/env bash
3e3af552# Grove - Import a Git repository into Mononoke
0d60b203# Uses the gitimport tool (inside the mononoke container) to import a Git repo
3e3af554# into Mononoke's blobstore.
3e3af555#
0d60b206# Usage: ./import-repo.sh <git-repo-path-or-url> <repo-name> [commit-sha]
0d60b207#
0d60b208# If commit-sha is provided, does incremental import (missing-for-commit).
0d60b209# Otherwise, does a full-repo import (first time only).
3e3af5510#
3e3af5511# Examples:
36387cc12# ./import-repo.sh /path/to/local/repo grove
3e3af5513# ./import-repo.sh https://github.com/user/repo.git myrepo
36387cc14# ./import-repo.sh /path/to/local/repo grove abc123 # incremental
3e3af5515
3e3af5516set -euo pipefail
3e3af5517
3e3af5518SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
3e3af5519GROVE_DIR="$SCRIPT_DIR/.."
0d60b2020HUB_DIR="$GROVE_DIR/hub"
0d60b2021DATA_DIR="${GROVE_DATA_DIR:-/data/grove}"
0d60b2022CONFIG_DIR="$DATA_DIR/mononoke-config"
3e3af5523
0d60b2024GIT_SOURCE="${1:?Usage: ./import-repo.sh <git-repo-path-or-url> <repo-name> [commit-sha]}"
0d60b2025REPO_NAME="${2:?Usage: ./import-repo.sh <git-repo-path-or-url> <repo-name> [commit-sha]}"
0d60b2026COMMIT_SHA="${3:-}"
3e3af5527
3e3af5528echo "=== Grove: Importing Git repo into Mononoke ==="
3e3af5529echo "Source: $GIT_SOURCE"
3e3af5530echo "Repo name: $REPO_NAME"
0d60b2031if [[ -n "$COMMIT_SHA" ]]; then
0d60b2032 echo "Mode: incremental (missing-for-commit $COMMIT_SHA)"
0d60b2033else
0d60b2034 echo "Mode: full-repo"
0d60b2035fi
3e3af5536echo ""
3e3af5537
3e3af5538# Step 1: Clone/prepare bare repo for import
0d60b2039BARE_REPO="$DATA_DIR/$REPO_NAME-bare.git"
0d60b2040
0d60b2041echo "Step 1: Preparing bare clone at $BARE_REPO..."
0d60b2042if [[ -d "$BARE_REPO" ]]; then
0d60b2043 echo " Bare repo already exists, updating..."
0d60b2044 if [[ -d "$GIT_SOURCE" ]]; then
0d60b2045 (cd "$GIT_SOURCE" && git push "$BARE_REPO" --all --force)
0d60b2046 elif [[ "$GIT_SOURCE" == http* ]] || [[ "$GIT_SOURCE" == git@* ]]; then
0d60b2047 git clone --bare "$GIT_SOURCE" "${BARE_REPO}.tmp"
0d60b2048 rm -rf "$BARE_REPO"
0d60b2049 mv "${BARE_REPO}.tmp" "$BARE_REPO"
0d60b2050 fi
3e3af5551else
0d60b2052 if [[ -d "$GIT_SOURCE" ]]; then
0d60b2053 git clone --bare "$GIT_SOURCE" "$BARE_REPO"
0d60b2054 elif [[ "$GIT_SOURCE" == http* ]] || [[ "$GIT_SOURCE" == git@* ]]; then
0d60b2055 git clone --bare "$GIT_SOURCE" "$BARE_REPO"
0d60b2056 else
0d60b2057 echo "Error: $GIT_SOURCE is not a valid git repo path or URL"
0d60b2058 exit 1
0d60b2059 fi
3e3af5560fi
3e3af5561echo " Bare clone ready at $BARE_REPO"
3e3af5562
3e3af5563# Step 2: Create repo config if it doesn't exist
3e3af5564REPO_DEF_DIR="$CONFIG_DIR/repo_definitions/$REPO_NAME"
3e3af5565REPO_CFG_DIR="$CONFIG_DIR/repos/$REPO_NAME"
3e3af5566
3e3af5567if [[ ! -d "$REPO_DEF_DIR" ]]; then
3e3af5568 echo ""
3e3af5569 echo "Step 2: Creating Mononoke config for '$REPO_NAME'..."
3e3af5570
3e3af5571 # Find next available repo_id
3e3af5572 MAX_ID=0
3e3af5573 for f in "$CONFIG_DIR"/repo_definitions/*/server.toml; do
3e3af5574 if [[ -f "$f" ]]; then
3e3af5575 ID=$(grep -oP 'repo_id\s*=\s*\K[0-9]+' "$f" 2>/dev/null || echo 0)
3e3af5576 if (( ID > MAX_ID )); then
3e3af5577 MAX_ID=$ID
3e3af5578 fi
3e3af5579 fi
3e3af5580 done
3e3af5581 NEXT_ID=$((MAX_ID + 1))
3e3af5582
3e3af5583 mkdir -p "$REPO_DEF_DIR" "$REPO_CFG_DIR"
3e3af5584
3e3af5585 cat > "$REPO_DEF_DIR/server.toml" <<EOF
3e3af5586repo_id = $NEXT_ID
3e3af5587repo_name = "$REPO_NAME"
3e3af5588repo_config = "$REPO_NAME"
3e3af5589enabled = true
6c9fcae90hipster_acl = "default"
3e3af5591EOF
3e3af5592
3e3af5593 cat > "$REPO_CFG_DIR/server.toml" <<EOF
3e3af5594storage_config = "default"
3e3af5595
0d60b2096[hook_manager_params]
0d60b2097disable_acl_checker = true
0d60b2098
3e3af5599[push]
3e3af55100pure_push_allowed = true
3e3af55101
3e3af55102[pushrebase]
3e3af55103rewritedates = false
3e3af55104
3e3af55105[source_control_service]
3e3af55106permit_writes = true
3e3af55107permit_service_writes = true
3e3af55108
6c9fcae109[git_configs.git_bundle_uri_config.uri_generator_type.local_fs]
6c9fcae110
0d60b20111[infinitepush]
0d60b20112allow_writes = true
0d60b20113
0d60b20114[commit_cloud_config]
0d60b20115
3e3af55116[derived_data_config]
3e3af55117enabled_config_name = "default"
3e3af55118
3e3af55119[derived_data_config.available_configs.default]
3e3af55120types = [
3e3af55121 "blame",
3e3af55122 "changeset_info",
3e3af55123 "fastlog",
3e3af55124 "filenodes",
3e3af55125 "fsnodes",
3e3af55126 "git_commits",
3e3af55127 "git_delta_manifests_v2",
6c9fcae128 "unodes",
3e3af55129 "hgchangesets",
3e3af55130 "skeleton_manifests",
1688ad1131 "skeleton_manifests_v2",
6c9fcae132 "ccsm",
3e3af55133]
0d60b20134
0d60b20135[derived_data_config.available_configs.default.git_delta_manifest_v2_config]
0d60b20136max_inlined_object_size = 2000
0d60b20137max_inlined_delta_size = 2000
0d60b20138delta_chunk_size = 1000000
3e3af55139EOF
3e3af55140
3e3af55141 echo " Config created with repo_id=$NEXT_ID"
0d60b20142 echo ""
0d60b20143 echo " NOTE: Mononoke must be restarted to pick up the new repo config."
0d60b20144 echo " Run: cd $HUB_DIR && docker compose restart mononoke-slapi grove-bridge mononoke-git"
0d60b20145 echo ""
0d60b20146 read -p " Restart Mononoke now? [y/N] " -n 1 -r
0d60b20147 echo
0d60b20148 if [[ $REPLY =~ ^[Yy]$ ]]; then
0d60b20149 echo " Restarting Mononoke services..."
0d60b20150 (cd "$HUB_DIR" && docker compose restart mononoke-slapi grove-bridge mononoke-git)
0d60b20151 echo " Waiting 5s for services to start..."
0d60b20152 sleep 5
0d60b20153 fi
3e3af55154else
3e3af55155 echo ""
3e3af55156 echo "Step 2: Config for '$REPO_NAME' already exists, skipping."
3e3af55157fi
3e3af55158
0d60b20159# Step 3: Run gitimport via docker compose
3e3af55160echo ""
3e3af55161echo "Step 3: Importing into Mononoke via gitimport..."
3e3af55162echo " This may take a while for large repos."
3e3af55163
0d60b20164IMPORT_MODE="full-repo"
0d60b20165IMPORT_ARGS=()
0d60b20166if [[ -n "$COMMIT_SHA" ]]; then
0d60b20167 IMPORT_MODE="missing-for-commit"
0d60b20168 IMPORT_ARGS=("$COMMIT_SHA")
0d60b20169fi
3e3af55170
0d60b20171(cd "$HUB_DIR" && docker compose run --rm --entrypoint gitimport grove-bridge \
0d60b20172 --repo-name "$REPO_NAME" \
0d60b20173 --config-path "$CONFIG_DIR" \
0d60b20174 --local-configerator-path "$DATA_DIR/configerator" \
0d60b20175 --cache-mode disabled \
0d60b20176 --just-knobs-config-path "$DATA_DIR/justknobs.json" \
0d60b20177 --generate-bookmarks \
0d60b20178 --derive-hg \
0d60b20179 --git-command-path /usr/bin/git \
0d60b20180 --concurrency 5 \
0d60b20181 "$BARE_REPO" \
0d60b20182 "$IMPORT_MODE" \
0d60b20183 "${IMPORT_ARGS[@]+"${IMPORT_ARGS[@]}"}")
3e3af55184
3e3af55185echo ""
3e3af55186echo "=== Import complete ==="
3e3af55187echo "Repo '$REPO_NAME' is now available in Mononoke."
0d60b20188echo ""
0d60b20189echo "Clone via git: git clone https://grove.host/repos/$REPO_NAME"
8d8e815190echo "Clone via grove: grove clone $REPO_NAME"