13.2 KB397 lines
Blame
1# syntax=docker/dockerfile:1
2# Grove: EdenFS Docker Build
3#
4# Build: cd /build/sapling && docker build -f /build/grove/docker/Dockerfile.edenfs -t grove/edenfs .
5# (context must be the sapling/ directory)
6#
7# Requires grove/sapling-deps:latest to be built first (shared C++ deps).
8# EdenFS is a C++/Rust hybrid (CMake build, not pure Cargo like Mononoke).
9
10# =============================================================================
11# Stage 1: Build EdenFS (C++ deps from shared base image)
12# =============================================================================
13FROM grove/sapling-deps:latest AS builder
14
15# Copy full source tree.
16COPY . /build
17
18# Fix shipit mismatches: the Facebook->OSS export renames/strips some files
19# but doesn't update CMakeLists.txt references.
20RUN ln -sf sl eden/fs/store/hg && \
21 touch eden/fs/inodes/InodeNumber.cpp eden/fs/service/PrettyPrinters.cpp && \
22 sed -i '/add_subdirectory(eden\/integration)/d; /add_subdirectory(eden\/test_support)/d' CMakeLists.txt && \
23 sed -i '/add_subdirectory(cli_rs\/edenfsctl)/d' eden/fs/CMakeLists.txt && \
24 cp eden/scm/lib/backingstore/include/ffi.h eden/scm/lib/backingstore/ && \
25 ln -sf include/SaplingBackingStoreError.h eden/scm/lib/backingstore/SaplingBackingStoreError.h && \
26 cp eden/scm/lib/edenfs_ffi/include/ffi.h eden/scm/lib/edenfs_ffi/ && \
27 sed -i '/edencommon::edencommon_utils/a\ backingstore' eden/fs/utils/CMakeLists.txt && \
28 sed -i '/file(GLOB PRIVHELPER_SRCS/a file(GLOB PRIORITY_SRCS "priority/ProcessPriority.cpp" "priority/LinuxMemoryPriority.cpp")\nlist(APPEND PRIVHELPER_SRCS ${PRIORITY_SRCS})' eden/fs/privhelper/CMakeLists.txt && \
29 sed -i '/edencommon::edencommon_os/a\ cpptoml' eden/scm/lib/backingstore/CMakeLists.txt && \
30 sed -i 's/^name = "sapling-backingstore"/name = "backingstore"/' eden/scm/lib/backingstore/Cargo.toml && \
31 sed -i 's/^name = "sapling-edenfs_ffi"/name = "edenfs_ffi"/' eden/scm/lib/edenfs_ffi/Cargo.toml && \
32 sed -i 's/sapling-backingstore/backingstore/g' eden/scm/lib/backingstore/benches/Cargo.toml && \
33 rm -f eden/scm/Cargo.lock && \
34 sed -i 's/"specialization", //' eden/scm/lib/backingstore/Cargo.toml && \
35 sed -i 's/^sapling-workingcopy = { version = "0.1.0", path = "..\/workingcopy", optional = true }/workingcopy = { package = "sapling-workingcopy", version = "0.1.0", path = "..\/workingcopy" }/' eden/scm/lib/repo/Cargo.toml && \
36 sed -i 's/^wdir = .*/wdir = []/' eden/scm/lib/repo/Cargo.toml && \
37 sed -i 's/sapling-repo = { version = "0.1.0", path = "..\/repo" }/sapling-repo = { version = "0.1.0", path = "..\/repo", features = ["wdir"] }/' eden/scm/lib/backingstore/Cargo.toml
38
39# Remove Meta-only OBC dependencies from OSS EdenFS sources.
40RUN <<'PATCH_OBC'
41python3 - <<'PY'
42from pathlib import Path
43
44
45def replace_exact(path: str, old: str, new: str) -> None:
46 p = Path(path)
47 s = p.read_text()
48 if old not in s:
49 raise SystemExit(f"pattern not found in {path}: {old[:80]!r}")
50 p.write_text(s.replace(old, new))
51
52
53# eden/fs/service: remove OBCAvg header and concrete metric type
54replace_exact(
55 "eden/fs/service/EdenServer.h",
56 '#include "monitoring/obc/OBCAvg.h"\n',
57 "",
58)
59replace_exact(
60 "eden/fs/service/EdenServer.h",
61 "#include <chrono>\n",
62 "#include <chrono>\n#include <cstdint>\n",
63)
64replace_exact(
65 "eden/fs/service/EdenServer.h",
66 " monitoring::OBCAvg memory_vm_rss_bytes_;\n",
67 " uint64_t memory_vm_rss_bytes_{0};\n",
68)
69replace_exact(
70 "eden/fs/service/EdenServer.cpp",
71 """ if (config->enableOBCOnEden.getValue()) {
72 // Get the hostname without the ".facebook.com" suffix
73 auto hostname = facebook::network::getLocalHost(/*stripFbDomain=*/true);
74 std::vector<std::string> entities;
75 auto reWorkerID = std::getenv("REMOTE_EXECUTION_WORKER");
76 if (reWorkerID == nullptr) {
77 entities = {hostname};
78 } else {
79 entities = {hostname, fmt::format("{}:{}", hostname, reWorkerID)};
80 }
81 memory_vm_rss_bytes_ = monitoring::OBCAvg(
82 monitoring::OdsCategoryId::ODS_EDEN,
83 fmt::format("eden.{}", kMemoryVmRssBytes),
84 entities);
85 // Report memory usage stats once every 60 seconds
86 memoryStatsTask_.updateInterval(60s);
87 }
88""",
89 """ if (config->enableOBCOnEden.getValue()) {
90 // OBC is Meta-internal; preserve memory stats reporting cadence in OSS.
91 memoryStatsTask_.updateInterval(60s);
92 }
93""",
94)
95
96# eden/fs/store/sl: remove OBCPxx header and concrete metric types
97replace_exact(
98 "eden/fs/store/sl/SaplingBackingStore.h",
99 '#include "monitoring/obc/OBCPxx.h"\n',
100 "",
101)
102replace_exact(
103 "eden/fs/store/sl/SaplingBackingStore.h",
104 "#include <atomic>\n",
105 "#include <atomic>\n#include <cstdint>\n",
106)
107replace_exact(
108 "eden/fs/store/sl/SaplingBackingStore.h",
109 " monitoring::OBCP99P95P50 getBlobPerRepoLatencies_; // calculates p50, p95, p99\n",
110 " uint64_t getBlobPerRepoLatencies_{0}; // OSS fallback counter\n",
111)
112replace_exact(
113 "eden/fs/store/sl/SaplingBackingStore.h",
114 " monitoring::OBCP99P95P50 getTreePerRepoLatencies_; // calculates p50, p95, p99\n",
115 " uint64_t getTreePerRepoLatencies_{0}; // OSS fallback counter\n",
116)
117replace_exact(
118 "eden/fs/store/sl/SaplingBackingStore.cpp",
119 """void SaplingBackingStore::initializeOBCCounters() {
120 // Get the hostname without the ".facebook.com" suffix
121 auto hostname = facebook::network::getLocalHost(/*stripFbDomain=*/true);
122 getBlobPerRepoLatencies_ = monitoring::OBCP99P95P50(
123 monitoring::OdsCategoryId::ODS_EDEN,
124 fmt::format("eden.store.sapling.fetch_blob_{}_us", repoName_),
125 {hostname});
126 getTreePerRepoLatencies_ = monitoring::OBCP99P95P50(
127 monitoring::OdsCategoryId::ODS_EDEN,
128 fmt::format("eden.store.sapling.fetch_tree_{}_us", repoName_),
129 {hostname});
130 isOBCEnabled_ = true;
131}
132""",
133 """void SaplingBackingStore::initializeOBCCounters() {
134 isOBCEnabled_ = true;
135}
136""",
137)
138PY
139PATCH_OBC
140
141# Additional OSS-only source patching:
142# - Disable redirect_ffi CXX bridge usage (lib.rs.h is not generated in CMake build)
143# - Avoid duplicate class definitions from copied backingstore headers
144RUN <<'PATCH_OSS'
145python3 - <<'PY'
146from pathlib import Path
147
148
149def replace_exact(path: str, old: str, new: str) -> None:
150 p = Path(path)
151 s = p.read_text()
152 if old not in s:
153 raise SystemExit(f"pattern not found in {path}: {old[:80]!r}")
154 p.write_text(s.replace(old, new))
155
156
157replace_exact(
158 "eden/fs/service/EdenServiceHandler.cpp",
159 '#include "eden/fs/rust/redirect_ffi/include/ffi.h"\n',
160 "",
161)
162replace_exact(
163 "eden/fs/service/EdenServiceHandler.cpp",
164 '#include "eden/fs/rust/redirect_ffi/src/lib.rs.h"\n',
165 "",
166)
167replace_exact(
168 "eden/fs/service/EdenServiceHandler.cpp",
169 """void EdenServiceHandler::listRedirections(
170 ListRedirectionsResponse& response,
171 std::unique_ptr<ListRedirectionsRequest> request) {
172 auto mountId = request->mount();
173 auto helper = INSTRUMENT_THRIFT_CALL(DBG3, *mountId);
174
175 const auto& configDir = server_->getEdenDir();
176 const auto& edenEtcDir =
177 server_->getServerState()->getEdenConfig()->getSystemConfigDir();
178
179 auto redirsFFI = list_redirections(
180 absolutePathFromThrift(*mountId->mountPoint()).stringWithoutUNC(),
181 configDir.stringWithoutUNC(),
182 edenEtcDir.stringWithoutUNC());
183
184 std::vector<Redirection> redirs(redirsFFI.size());
185 std::transform(
186 redirsFFI.begin(), redirsFFI.end(), redirs.begin(), [](auto&& redirFFI) {
187 return redirectionFromFFI(std::move(redirFFI));
188 });
189
190 response.redirections() = std::move(redirs);
191}
192""",
193 """void EdenServiceHandler::listRedirections(
194 ListRedirectionsResponse& response,
195 std::unique_ptr<ListRedirectionsRequest> request) {
196 auto mountId = request->mount();
197 auto helper = INSTRUMENT_THRIFT_CALL(DBG3, *mountId);
198 (void)request;
199 response.redirections() = std::vector<Redirection>{};
200}
201""",
202)
203replace_exact(
204 "eden/fs/store/ObjectStore.cpp",
205 " co_return std::move(result.blob);\n",
206 " std::shared_ptr<const Blob> blob = result.blob.get();\n"
207 " co_return blob;\n",
208)
209PY
210PATCH_OSS
211
212# Fix ambiguous Cargo.toml deps (both path and git specified — shipit bug)
213RUN <<'FIX_CARGO'
214python3 -c "
215import re, glob
216for f in glob.glob('eden/fs/cli_rs/*/Cargo.toml'):
217 with open(f) as fh: c = fh.read()
218 c = re.sub(r', git = \"[^\"]+\", branch = \"[^\"]+\"', '', c)
219 with open(f, 'w') as fh: fh.write(c)
220"
221FIX_CARGO
222
223# Create minimal iobuf stub crate — wraps folly::IOBuf for the CXX bridge.
224# The real iobuf crate is Meta-internal; this stub provides just enough for
225# the backingstore FFI to compile. folly::IOBuf is available from the C++ deps.
226RUN mkdir -p eden/scm/lib/iobuf/src
227RUN <<'IOBUF_CARGO'
228cat > eden/scm/lib/iobuf/Cargo.toml << 'INNER'
229[package]
230name = "iobuf"
231version = "0.1.0"
232edition = "2021"
233
234[dependencies]
235cxx = "1.0.119"
236INNER
237IOBUF_CARGO
238RUN <<'IOBUF_RS'
239cat > eden/scm/lib/iobuf/src/lib.rs << 'INNER'
240use std::fmt;
241
242#[cxx::bridge(namespace = "folly")]
243mod ffi {
244 unsafe extern "C++" {
245 type IOBuf;
246 }
247 impl UniquePtr<IOBuf> {}
248}
249
250pub use ffi::IOBuf;
251
252#[derive(Clone)]
253pub struct IOBufShared {
254 data: Vec<u8>,
255}
256
257impl fmt::Debug for IOBufShared {
258 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
259 f.debug_struct("IOBufShared").field("len", &self.data.len()).finish()
260 }
261}
262
263impl IOBufShared {
264 pub unsafe fn from_owner<T: AsRef<[u8]>>(owner: T) -> Self {
265 Self { data: owner.as_ref().to_vec() }
266 }
267 pub fn append_to_end(&mut self, other: IOBufShared) {
268 self.data.extend_from_slice(&other.data);
269 }
270 pub fn len(&self) -> usize { self.data.len() }
271 pub fn is_empty(&self) -> bool { self.data.is_empty() }
272}
273
274impl From<&str> for IOBufShared {
275 fn from(s: &str) -> Self { Self { data: s.as_bytes().to_vec() } }
276}
277
278impl From<IOBufShared> for Vec<u8> {
279 fn from(buf: IOBufShared) -> Vec<u8> { buf.data }
280}
281
282impl PartialEq for IOBufShared {
283 fn eq(&self, other: &Self) -> bool { self.data == other.data }
284}
285INNER
286IOBUF_RS
287# Add iobuf to workspace + deps, and patch source for OSS compatibility.
288RUN <<'PATCH_SCRIPT'
289#!/bin/bash
290set -e
291
292# Add iobuf to workspace members
293sed -i '/"lib\/identity",/a\ "lib/iobuf",' eden/scm/Cargo.toml
294
295# Add iobuf dep to backingstore
296sed -i '/^cxx = /a iobuf = { version = "0.1.0", path = "../iobuf" }' eden/scm/lib/backingstore/Cargo.toml
297
298# Add iobuf dep to blob
299sed -i '/^sha1 = /a iobuf = { version = "0.1.0", path = "../iobuf" }' eden/scm/lib/blob/Cargo.toml
300
301# Patch ffi.rs: add impl UniquePtr<IOBuf> {}, add make_iobuf_from_bytes fn,
302# replace into_iobuf().into() calls
303python3 -c "
304import re
305with open('eden/scm/lib/backingstore/src/ffi.rs', 'r') as f:
306 content = f.read()
307
308# Add make_iobuf_from_bytes fn to the extern C++ block containing IOBuf
309content = content.replace(
310 'type IOBuf = iobuf::IOBuf;',
311 'type IOBuf = iobuf::IOBuf;\n fn make_iobuf_from_bytes(data: &[u8]) -> UniquePtr<IOBuf>;'
312)
313
314# Replace blob.into_iobuf().into() with our helper
315content = content.replace(
316 'blob.into_iobuf().into()',
317 'ffi::make_iobuf_from_bytes(&blob.into_vec())'
318)
319
320with open('eden/scm/lib/backingstore/src/ffi.rs', 'w') as f:
321 f.write(content)
322"
323
324# Add C++ helper to create IOBuf from bytes
325cat >> eden/scm/lib/backingstore/include/ffi.h << 'CPP_EOF'
326
327#include <folly/io/IOBuf.h>
328namespace sapling {
329inline std::unique_ptr<folly::IOBuf> make_iobuf_from_bytes(
330 rust::Slice<const uint8_t> data) {
331 return folly::IOBuf::copyBuffer(data.data(), data.length());
332}
333} // namespace sapling
334CPP_EOF
335PATCH_SCRIPT
336
337# Build EdenFS using getdeps. Unlike Mononoke (pure Cargo), EdenFS uses CMake
338# which needs proper workspace resolution for the Rust FFI crates.
339RUN --mount=type=cache,target=/root/.cargo/registry \
340 python3 build/fbcode_builder/getdeps.py --allow-system-packages \
341 build --no-deps --build-type MinSizeRel --no-tests --src-dir=. eden \
342 --project-install-prefix eden:/
343
344# Collect artifacts with dynamic library fixups
345RUN python3 build/fbcode_builder/getdeps.py --allow-system-packages \
346 fixup-dyn-deps --strip --src-dir=. eden \
347 /artifacts --project-install-prefix eden:/ \
348 --final-install-prefix /usr/local
349
350# =============================================================================
351# Stage 2: Minimal runtime image
352# =============================================================================
353FROM ubuntu:22.04 AS runtime
354
355RUN apt-get update && apt-get install -y \
356 ca-certificates \
357 git \
358 libssl3 \
359 zlib1g \
360 libzstd1 \
361 liblz4-1 \
362 libsnappy1v5 \
363 libsodium23 \
364 libevent-2.1-7 \
365 libdouble-conversion3 \
366 libgflags2.2 \
367 libgoogle-glog0v5 \
368 libunwind8 \
369 libdwarf1 \
370 libfuse2 \
371 libre2-9 \
372 libgit2-1.1 \
373 liblmdb0 \
374 libsqlite3-0 \
375 libcurl4 \
376 python3 \
377 && rm -rf /var/lib/apt/lists/*
378
379# Copy built artifacts (fixup-dyn-deps puts binaries in /artifacts/bin/)
380COPY --from=builder /artifacts/bin /usr/local/bin
381
382# Create data directories
383RUN mkdir -p /data/eden /config /certs
384
385# EdenFS uses FUSE on Linux — needs /dev/fuse access at runtime
386# (requires --privileged or --device /dev/fuse when running the container)
387
388# edenfs daemon port (thrift)
389EXPOSE 8400
390
391# Default entrypoint runs the edenfs daemon
392ENTRYPOINT ["/usr/local/bin/edenfs"]
393CMD ["--edenDir", "/data/eden", \
394 "--etcEdenDir", "/config", \
395 "--configPath", "/config/edenfs.toml", \
396 "--foreground"]
397