grove init: preserve full git history via server-side gitimport

Instead of squashing all git commits into one, grove init now:
- Creates a bare clone and uploads it to the API
- Server runs gitimport to import full commit history into Mononoke
- CLI clones from Grove to set up Sapling working copy

Also fixes .git.bak being included in the sl commit.
Anton Kaminsky29d ago90d5eb8a0789parent c7b7f92
6 files changed+354-117
api/package.json
@@ -13,6 +13,7 @@
1313 "dependencies": {
1414 "@fastify/cors": "^11.0.0",
1515 "@fastify/jwt": "^9.0.0",
16 "@fastify/multipart": "^9.4.0",
1617 "@fastify/static": "^8.1.0",
1718 "better-sqlite3": "^11.7.0",
1819 "fastify": "^5.2.0",
1920
api/src/routes/repos.ts
@@ -1,6 +1,9 @@
11import type { FastifyInstance, FastifyRequest, FastifyReply } from "fastify";
22import { z } from "zod";
3import { spawn } from "child_process";
3import { spawn, execSync } from "child_process";
4import { createWriteStream } from "fs";
5import { mkdir, rm } from "fs/promises";
6import { pipeline } from "stream/promises";
47import { BridgeService } from "../services/bridge.js";
58import type { MononokeProvisioner } from "../services/mononoke-provisioner.js";
69import { optionalAuth } from "../auth/middleware.js";
@@ -757,6 +760,131 @@
757760 reply.raw.end();
758761 }
759762 );
763
764 // Import a Git repository from an uploaded bare repo tarball (SSE progress stream)
765 app.post<{ Params: { owner: string; repo: string } }>(
766 "/:owner/:repo/import-bundle",
767 {
768 preHandler: [(app as any).authenticate],
769 },
770 async (request, reply) => {
771 const { owner, repo: repoName } = request.params;
772 const db = (app as any).db;
773
774 // Verify repo exists
775 const repoRow = db
776 .prepare(`SELECT * FROM repos_with_owner WHERE owner_name = ? AND name = ?`)
777 .get(owner, repoName) as any;
778 if (!repoRow) {
779 return reply.code(404).send({ error: "Repository not found" });
780 }
781
782 // Read the uploaded file
783 const file = await (request as any).file();
784 if (!file) {
785 return reply.code(400).send({ error: "No file uploaded" });
786 }
787
788 // SSE stream
789 reply.raw.writeHead(200, {
790 "Content-Type": "text/event-stream",
791 "Cache-Control": "no-cache",
792 Connection: "keep-alive",
793 });
794
795 const send = (event: string, data: any) => {
796 reply.raw.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
797 };
798
799 const bareRepo = `${DATA_DIR}/${repoName}-bare.git`;
800 const tarPath = `${DATA_DIR}/${repoName}-bare.tar.gz`;
801
802 try {
803 // Step 1: Save uploaded tarball and extract
804 send("progress", { step: "upload", message: "Receiving bare repo..." });
805
806 await pipeline(file.file, createWriteStream(tarPath));
807
808 send("progress", { step: "upload", message: "Extracting..." });
809
810 await rm(bareRepo, { recursive: true, force: true });
811 await mkdir(bareRepo, { recursive: true });
812
813 await runDocker([
814 "run", "--rm",
815 "-v", "/data/grove:/data/grove",
816 "grove/mononoke:latest",
817 "tar", "xzf", tarPath, "-C", `${DATA_DIR}`,
818 ], (line) => {
819 send("log", { step: "upload", line });
820 });
821
822 // The tar extracts as bare.git/ — rename to match expected path
823 try {
824 execSync(`mv ${DATA_DIR}/bare.git ${bareRepo}`, { stdio: "pipe" });
825 } catch {
826 // Already at the right path (tarball used repo name)
827 }
828
829 send("progress", { step: "upload", message: "Extracted." });
830
831 // Step 2: gitimport into Mononoke
832 send("progress", { step: "import", message: "Importing into Mononoke..." });
833
834 await runDocker([
835 "run", "--rm",
836 "-v", "/data/grove:/data/grove",
837 "--entrypoint", "gitimport",
838 "grove/mononoke:latest",
839 "--repo-name", repoName,
840 "--config-path", MONONOKE_CONFIG_PATH,
841 "--local-configerator-path", `${DATA_DIR}/configerator`,
842 "--cache-mode", "disabled",
843 "--just-knobs-config-path", `${DATA_DIR}/justknobs.json`,
844 "--generate-bookmarks",
845 "--derive-hg",
846 "--git-command-path", "/usr/bin/git",
847 "--concurrency", "5",
848 bareRepo,
849 "full-repo",
850 ], (line) => {
851 send("log", { step: "import", line });
852 });
853
854 send("progress", { step: "import", message: "Import complete." });
855
856 // Step 3: Restart Mononoke services
857 send("progress", { step: "restart", message: "Restarting services..." });
858
859 const provisioner = (app as any).mononokeProvisioner as MononokeProvisioner;
860 await provisioner.restartMononoke();
861
862 send("progress", { step: "restart", message: "Services restarted." });
863
864 // Clean up
865 await runDocker([
866 "run", "--rm",
867 "-v", "/data/grove:/data/grove",
868 "grove/mononoke:latest",
869 "rm", "-rf", bareRepo, tarPath,
870 ], () => {});
871
872 send("done", { success: true });
873 } catch (err: any) {
874 // Clean up on error
875 await runDocker([
876 "run", "--rm",
877 "-v", "/data/grove:/data/grove",
878 "grove/mononoke:latest",
879 "rm", "-rf", bareRepo, tarPath,
880 ], () => {}).catch(() => {});
881
882 send("error", { message: err.message ?? "Import failed" });
883 }
884
885 reply.raw.end();
886 }
887 );
760888}
761889
762890/**
763891
api/src/server.ts
@@ -1,6 +1,7 @@
11import Fastify from "fastify";
22import cors from "@fastify/cors";
33import jwt from "@fastify/jwt";
4import multipart from "@fastify/multipart";
45import type Database from "better-sqlite3";
56import { initDatabase } from "./services/database.js";
67import { repoRoutes } from "./routes/repos.js";
@@ -35,6 +36,10 @@
3536 secret: process.env.JWT_SECRET ?? "grove-dev-secret",
3637});
3738
39await app.register(multipart, {
40 limits: { fileSize: 500 * 1024 * 1024 }, // 500MB
41});
42
3843// Initialize database
3944const db = initDatabase(
4045 process.env.DATABASE_PATH ?? "./data/grove.db"
4146
cli/src/api.ts
@@ -1,4 +1,6 @@
11import { log } from "@clack/prompts";
2import { readFileSync } from "node:fs";
3import { basename } from "node:path";
24import { getHub, getToken } from "./config.js";
35
46export async function hubRequest<T>(
@@ -32,3 +34,67 @@
3234
3335 return res.json() as Promise<T>;
3436}
37
38/**
39 * Upload a file to a hub endpoint and stream SSE events back.
40 * Used for import-bundle which streams gitimport progress.
41 */
42export async function hubUploadStream(
43 path: string,
44 filePath: string,
45 onEvent: (event: string, data: any) => void,
46): Promise<void> {
47 const hub = await getHub();
48 const token = await getToken();
49
50 const fileData = readFileSync(filePath);
51 const blob = new Blob([fileData], { type: "application/gzip" });
52
53 const form = new FormData();
54 form.append("file", blob, basename(filePath));
55
56 const res = await fetch(`${hub}${path}`, {
57 method: "POST",
58 headers: {
59 Authorization: `Bearer ${token}`,
60 },
61 body: form,
62 });
63
64 if (!res.ok && !res.headers.get("content-type")?.includes("text/event-stream")) {
65 const body = await res.text();
66 throw new Error(`Upload failed (${res.status}): ${body}`);
67 }
68
69 // Parse SSE stream
70 const reader = res.body!.getReader();
71 const decoder = new TextDecoder();
72 let buffer = "";
73
74 while (true) {
75 const { done, value } = await reader.read();
76 if (done) break;
77
78 buffer += decoder.decode(value, { stream: true });
79 const lines = buffer.split("\n");
80 buffer = lines.pop()!; // keep incomplete line in buffer
81
82 let currentEvent = "";
83 for (const line of lines) {
84 if (line.startsWith("event: ")) {
85 currentEvent = line.slice(7);
86 } else if (line.startsWith("data: ")) {
87 try {
88 const data = JSON.parse(line.slice(6));
89 onEvent(currentEvent, data);
90 if (currentEvent === "error") {
91 throw new Error(data.message ?? "Import failed");
92 }
93 } catch (e) {
94 if (e instanceof SyntaxError) continue; // skip malformed JSON
95 throw e;
96 }
97 }
98 }
99 }
100}
35101
cli/src/commands/init.ts
@@ -3,7 +3,7 @@
33import { writeFileSync, existsSync, mkdirSync, renameSync, rmSync } from "node:fs";
44import { join, basename, resolve } from "node:path";
55import { homedir } from "node:os";
6import { hubRequest } from "../api.js";
6import { hubRequest, hubUploadStream } from "../api.js";
77import { getHub } from "../config.js";
88
99interface Repo {
@@ -168,12 +168,6 @@
168168 // fall back to main
169169 }
170170
171 // Get latest git commit message for the import commit
172 let lastCommitMsg = "Import from git";
173 try {
174 lastCommitMsg = execSync("git log -1 --format=%s", { cwd: dir, stdio: "pipe" }).toString().trim();
175 } catch {}
176
177171 // Count commits for user feedback
178172 let commitCount = "?";
179173 try {
@@ -181,42 +175,82 @@
181175 } catch {}
182176 log.info(`Found git repository with ${commitCount} commits on ${gitBranch}`);
183177
184 // Create Grove repo without seeding (we'll push the current tree)
178 // Create Grove repo without seeding
185179 const repo = await createRepo(name, owner, description, isPrivate, true);
186180 const hub = await getHub();
187 const config = buildSlConfig({ ...repo, default_branch: gitBranch }, hub);
181 const ownerName = repo.owner_name;
182
183 // Create bare clone and tar it up for upload
184 const tmpDir = join(dir, "..", `.grove-import-${name}-${Date.now()}`);
185 const bareDir = join(tmpDir, "bare.git");
186 const tarPath = join(tmpDir, "bare.tar.gz");
188187
189 // Rename .git out of the way so sl init doesn't conflict
190188 const s1 = spinner();
191 s1.start("Converting to Sapling repository");
192 const gitBackup = join(dir, ".git.bak");
193 renameSync(join(dir, ".git"), gitBackup);
194
195 // Init Sapling and commit all current files
196 execSync(`sl init --config init.prefer-git=false --config format.use-remotefilelog=true "${dir}"`, { stdio: "pipe" });
197 writeFileSync(join(dir, ".sl", "config"), config);
198 execSync(`sl add`, { cwd: dir, stdio: "pipe" });
199 execSync(`sl commit -m "Import from git (${commitCount} commits)\n\nLast git commit: ${lastCommitMsg}"`, { cwd: dir, stdio: "pipe" });
200 s1.stop("Sapling repository initialized with current tree");
201
202 // Push to Grove
189 s1.start("Preparing git history for import");
190 try {
191 mkdirSync(tmpDir, { recursive: true });
192 execSync(`git clone --bare "${dir}" "${bareDir}"`, { stdio: "pipe" });
193 execSync(`tar czf "${tarPath}" -C "${tmpDir}" bare.git`, { stdio: "pipe" });
194 s1.stop("Bare clone ready");
195 } catch (e: any) {
196 s1.stop("Failed to create bare clone");
197 log.error(e.stderr?.toString() || e.message);
198 rmSync(tmpDir, { recursive: true, force: true });
199 process.exit(1);
200 }
201
202 // Upload to server and run gitimport
203203 const s2 = spinner();
204 s2.start(`Pushing to ${gitBranch}`);
204 s2.start("Importing git history into Grove");
205205 try {
206 execSync(`sl push --to ${gitBranch} --create`, { cwd: dir, stdio: "pipe" });
206 await hubUploadStream(
207 `/api/repos/${ownerName}/${name}/import-bundle`,
208 tarPath,
209 (event, data) => {
210 if (event === "progress" && data.message) {
211 s2.message(data.message);
212 }
213 },
214 );
215 s2.stop("Git history imported");
216 } catch (e: any) {
217 s2.stop("Import failed");
218 log.error(e.message);
219 rmSync(tmpDir, { recursive: true, force: true });
220 process.exit(1);
221 }
222
223 // Remove temp upload files
224 rmSync(tmpDir, { recursive: true, force: true });
225
226 // Set up Sapling working copy by cloning from Grove
227 const s3 = spinner();
228 s3.start("Setting up Sapling working copy");
229 const cloneTmp = join(dir, "..", `.grove-clone-${name}-${Date.now()}`);
230 try {
231 // Clone into a temp dir, then move .sl into the working dir
232 execSync(`sl clone slapi:${name} "${cloneTmp}"`, { stdio: "pipe" });
233
234 // Remove .git and replace with .sl
235 rmSync(join(dir, ".git"), { recursive: true, force: true });
236 renameSync(join(cloneTmp, ".sl"), join(dir, ".sl"));
237
238 // Write config with correct remote settings
239 const config = buildSlConfig({ ...repo, default_branch: gitBranch }, hub);
240 writeFileSync(join(dir, ".sl", "config"), config);
241
242 s3.stop("Sapling working copy ready");
207243 } catch (e: any) {
208 s2.stop("Push failed");
244 s3.stop("Clone failed");
209245 log.error(e.stderr?.toString() || e.message);
210 // Restore .git so the user isn't left in a broken state
211 rmSync(join(dir, ".sl"), { recursive: true, force: true });
212 renameSync(gitBackup, join(dir, ".git"));
246 rmSync(cloneTmp, { recursive: true, force: true });
213247 process.exit(1);
214248 }
215 s2.stop(`Pushed to ${gitBranch}`);
216249
217 log.info(`Git data saved to .git.bak (safe to delete)`);
250 // Clean up clone temp dir
251 rmSync(cloneTmp, { recursive: true, force: true });
218252
219 outro(`Imported ${repo.owner_name}/${repo.name} from git`);
253 outro(`Imported ${ownerName}/${name} with full git history`);
220254}
221255
222256async function initFresh(dir: string, name: string, owner: string | undefined, description: string | undefined, isPrivate: boolean) {
223257
package-lock.json
@@ -425,6 +425,7 @@
425425 "dependencies": {
426426 "@fastify/cors": "^11.0.0",
427427 "@fastify/jwt": "^9.0.0",
428 "@fastify/multipart": "^9.4.0",
428429 "@fastify/static": "^8.1.0",
429430 "better-sqlite3": "^11.7.0",
430431 "fastify": "^5.2.0",
@@ -492,20 +493,6 @@
492493 "toad-cache": "^3.7.0"
493494 }
494495 },
495 "api/node_modules/@fastify/error": {
496 "version": "4.2.0",
497 "funding": [
498 {
499 "type": "github",
500 "url": "https://github.com/sponsors/fastify"
501 },
502 {
503 "type": "opencollective",
504 "url": "https://opencollective.com/fastify"
505 }
506 ],
507 "license": "MIT"
508 },
509496 "api/node_modules/@fastify/fast-json-stringify-compiler": {
510497 "version": "5.0.3",
511498 "funding": [
@@ -867,20 +854,6 @@
867854 "toad-cache": "^3.7.0"
868855 }
869856 },
870 "api/node_modules/fastify-plugin": {
871 "version": "5.1.0",
872 "funding": [
873 {
874 "type": "github",
875 "url": "https://github.com/sponsors/fastify"
876 },
877 {
878 "type": "opencollective",
879 "url": "https://opencollective.com/fastify"
880 }
881 ],
882 "license": "MIT"
883 },
884857 "api/node_modules/fastparallel": {
885858 "version": "2.4.1",
886859 "license": "ISC",
@@ -1199,20 +1172,6 @@
11991172 "node": ">=10"
12001173 }
12011174 },
1202 "api/node_modules/secure-json-parse": {
1203 "version": "4.1.0",
1204 "funding": [
1205 {
1206 "type": "github",
1207 "url": "https://github.com/sponsors/fastify"
1208 },
1209 {
1210 "type": "opencollective",
1211 "url": "https://opencollective.com/fastify"
1212 }
1213 ],
1214 "license": "BSD-3-Clause"
1215 },
12161175 "api/node_modules/semver": {
12171176 "version": "7.7.4",
12181177 "license": "ISC",
@@ -1376,20 +1335,6 @@
13761335 "toad-cache": "^3.7.0"
13771336 }
13781337 },
1379 "hub-api/node_modules/@fastify/error": {
1380 "version": "4.2.0",
1381 "funding": [
1382 {
1383 "type": "github",
1384 "url": "https://github.com/sponsors/fastify"
1385 },
1386 {
1387 "type": "opencollective",
1388 "url": "https://opencollective.com/fastify"
1389 }
1390 ],
1391 "license": "MIT"
1392 },
13931338 "hub-api/node_modules/@fastify/fast-json-stringify-compiler": {
13941339 "version": "5.0.3",
13951340 "funding": [
@@ -1880,20 +1825,6 @@
18801825 "toad-cache": "^3.7.0"
18811826 }
18821827 },
1883 "hub-api/node_modules/fastify-plugin": {
1884 "version": "5.1.0",
1885 "funding": [
1886 {
1887 "type": "github",
1888 "url": "https://github.com/sponsors/fastify"
1889 },
1890 {
1891 "type": "opencollective",
1892 "url": "https://opencollective.com/fastify"
1893 }
1894 ],
1895 "license": "MIT"
1896 },
18971828 "hub-api/node_modules/fastparallel": {
18981829 "version": "2.4.1",
18991830 "license": "ISC",
@@ -2146,20 +2077,6 @@
21462077 "node": ">=10"
21472078 }
21482079 },
2149 "hub-api/node_modules/secure-json-parse": {
2150 "version": "4.1.0",
2151 "funding": [
2152 {
2153 "type": "github",
2154 "url": "https://github.com/sponsors/fastify"
2155 },
2156 {
2157 "type": "opencollective",
2158 "url": "https://opencollective.com/fastify"
2159 }
2160 ],
2161 "license": "BSD-3-Clause"
2162 },
21632080 "hub-api/node_modules/semver": {
21642081 "version": "7.7.4",
21652082 "license": "ISC",
@@ -3413,9 +3330,63 @@
34133330 },
34143331 "node_modules/@fastify/busboy": {
34153332 "version": "3.2.0",
3416 "dev": true,
34173333 "license": "MIT"
34183334 },
3335 "node_modules/@fastify/deepmerge": {
3336 "version": "3.2.1",
3337 "resolved": "https://registry.npmjs.org/@fastify/deepmerge/-/deepmerge-3.2.1.tgz",
3338 "integrity": "sha512-N5Oqvltoa2r9z1tbx4xjky0oRR60v+T47Ic4J1ukoVQcptLOrIdRnCSdTGmOmajZuHVKlTnfcmrjyqsGEW1ztA==",
3339 "funding": [
3340 {
3341 "type": "github",
3342 "url": "https://github.com/sponsors/fastify"
3343 },
3344 {
3345 "type": "opencollective",
3346 "url": "https://opencollective.com/fastify"
3347 }
3348 ],
3349 "license": "MIT"
3350 },
3351 "node_modules/@fastify/error": {
3352 "version": "4.2.0",
3353 "resolved": "https://registry.npmjs.org/@fastify/error/-/error-4.2.0.tgz",
3354 "integrity": "sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ==",
3355 "funding": [
3356 {
3357 "type": "github",
3358 "url": "https://github.com/sponsors/fastify"
3359 },
3360 {
3361 "type": "opencollective",
3362 "url": "https://opencollective.com/fastify"
3363 }
3364 ],
3365 "license": "MIT"
3366 },
3367 "node_modules/@fastify/multipart": {
3368 "version": "9.4.0",
3369 "resolved": "https://registry.npmjs.org/@fastify/multipart/-/multipart-9.4.0.tgz",
3370 "integrity": "sha512-Z404bzZeLSXTBmp/trCBuoVFX28pM7rhv849Q5TsbTFZHuk1lc4QjQITTPK92DKVpXmNtJXeHSSc7GYvqFpxAQ==",
3371 "funding": [
3372 {
3373 "type": "github",
3374 "url": "https://github.com/sponsors/fastify"
3375 },
3376 {
3377 "type": "opencollective",
3378 "url": "https://opencollective.com/fastify"
3379 }
3380 ],
3381 "license": "MIT",
3382 "dependencies": {
3383 "@fastify/busboy": "^3.0.0",
3384 "@fastify/deepmerge": "^3.0.0",
3385 "@fastify/error": "^4.0.0",
3386 "fastify-plugin": "^5.0.0",
3387 "secure-json-parse": "^4.0.0"
3388 }
3389 },
34193390 "node_modules/@graphql-codegen/add": {
34203391 "version": "3.2.3",
34213392 "dev": true,
@@ -9863,6 +9834,22 @@
98639834 ],
98649835 "license": "BSD-3-Clause"
98659836 },
9837 "node_modules/fastify-plugin": {
9838 "version": "5.1.0",
9839 "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-5.1.0.tgz",
9840 "integrity": "sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw==",
9841 "funding": [
9842 {
9843 "type": "github",
9844 "url": "https://github.com/sponsors/fastify"
9845 },
9846 {
9847 "type": "opencollective",
9848 "url": "https://opencollective.com/fastify"
9849 }
9850 ],
9851 "license": "MIT"
9852 },
98669853 "node_modules/fastq": {
98679854 "version": "1.20.1",
98689855 "license": "ISC",
@@ -14975,6 +14962,22 @@
1497514962 "dev": true,
1497614963 "license": "MIT"
1497714964 },
14965 "node_modules/secure-json-parse": {
14966 "version": "4.1.0",
14967 "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz",
14968 "integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==",
14969 "funding": [
14970 {
14971 "type": "github",
14972 "url": "https://github.com/sponsors/fastify"
14973 },
14974 {
14975 "type": "opencollective",
14976 "url": "https://opencollective.com/fastify"
14977 }
14978 ],
14979 "license": "BSD-3-Clause"
14980 },
1497814981 "node_modules/semver": {
1497914982 "version": "6.3.1",
1498014983 "license": "ISC",
1498114984