docker/Dockerfile.edenfsblame
View source
99f1a2e1# syntax=docker/dockerfile:1
99f1a2e2# Grove: EdenFS Docker Build
99f1a2e3#
99f1a2e4# Build: cd /build/sapling && docker build -f /build/grove/docker/Dockerfile.edenfs -t grove/edenfs .
99f1a2e5# (context must be the sapling/ directory)
99f1a2e6#
2d46ffc7# Requires grove/sapling-deps:latest to be built first (shared C++ deps).
2d46ffc8# EdenFS is a C++/Rust hybrid (CMake build, not pure Cargo like Mononoke).
818dc909
99f1a2e10# =============================================================================
2d46ffc11# Stage 1: Build EdenFS (C++ deps from shared base image)
99f1a2e12# =============================================================================
2d46ffc13FROM grove/sapling-deps:latest AS builder
99f1a2e14
4dfd09b15# Copy full source tree.
99f1a2e16COPY . /build
99f1a2e17
2d46ffc18# Fix shipit mismatches: the Facebook->OSS export renames/strips some files
818dc9019# but doesn't update CMakeLists.txt references.
4dfd09b20RUN ln -sf sl eden/fs/store/hg && \
4dfd09b21 touch eden/fs/inodes/InodeNumber.cpp eden/fs/service/PrettyPrinters.cpp && \
4dfd09b22 sed -i '/add_subdirectory(eden\/integration)/d; /add_subdirectory(eden\/test_support)/d' CMakeLists.txt && \
36387cc23 sed -i '/add_subdirectory(cli_rs\/edenfsctl)/d' eden/fs/CMakeLists.txt && \
679ea0824 cp eden/scm/lib/backingstore/include/ffi.h eden/scm/lib/backingstore/ && \
5adbed725 ln -sf include/SaplingBackingStoreError.h eden/scm/lib/backingstore/SaplingBackingStoreError.h && \
679ea0826 cp eden/scm/lib/edenfs_ffi/include/ffi.h eden/scm/lib/edenfs_ffi/ && \
4dfd09b27 sed -i '/edencommon::edencommon_utils/a\ backingstore' eden/fs/utils/CMakeLists.txt && \
8dd15af28 sed -i '/file(GLOB PRIVHELPER_SRCS/a file(GLOB PRIORITY_SRCS "priority/ProcessPriority.cpp" "priority/LinuxMemoryPriority.cpp")\nlist(APPEND PRIVHELPER_SRCS ${PRIORITY_SRCS})' eden/fs/privhelper/CMakeLists.txt && \
36387cc29 sed -i '/edencommon::edencommon_os/a\ cpptoml' eden/scm/lib/backingstore/CMakeLists.txt && \
6e3776030 sed -i 's/^name = "sapling-backingstore"/name = "backingstore"/' eden/scm/lib/backingstore/Cargo.toml && \
6e3776031 sed -i 's/^name = "sapling-edenfs_ffi"/name = "edenfs_ffi"/' eden/scm/lib/edenfs_ffi/Cargo.toml && \
095047032 sed -i 's/sapling-backingstore/backingstore/g' eden/scm/lib/backingstore/benches/Cargo.toml && \
095047033 rm -f eden/scm/Cargo.lock && \
4dfd09b34 sed -i 's/"specialization", //' eden/scm/lib/backingstore/Cargo.toml && \
b0e115a35 sed -i 's/^sapling-workingcopy = { version = "0.1.0", path = "..\/workingcopy", optional = true }/workingcopy = { package = "sapling-workingcopy", version = "0.1.0", path = "..\/workingcopy" }/' eden/scm/lib/repo/Cargo.toml && \
2b7f0d936 sed -i 's/^wdir = .*/wdir = []/' eden/scm/lib/repo/Cargo.toml && \
2b7f0d937 sed -i 's/sapling-repo = { version = "0.1.0", path = "..\/repo" }/sapling-repo = { version = "0.1.0", path = "..\/repo", features = ["wdir"] }/' eden/scm/lib/backingstore/Cargo.toml
4dfd09b38
e9b652839# Remove Meta-only OBC dependencies from OSS EdenFS sources.
e9b652840RUN <<'PATCH_OBC'
e9b652841python3 - <<'PY'
e9b652842from pathlib import Path
e9b652843
e9b652844
e9b652845def replace_exact(path: str, old: str, new: str) -> None:
e9b652846 p = Path(path)
e9b652847 s = p.read_text()
e9b652848 if old not in s:
e9b652849 raise SystemExit(f"pattern not found in {path}: {old[:80]!r}")
e9b652850 p.write_text(s.replace(old, new))
e9b652851
e9b652852
e9b652853# eden/fs/service: remove OBCAvg header and concrete metric type
e9b652854replace_exact(
e9b652855 "eden/fs/service/EdenServer.h",
e9b652856 '#include "monitoring/obc/OBCAvg.h"\n',
e9b652857 "",
e9b652858)
e9b652859replace_exact(
e9b652860 "eden/fs/service/EdenServer.h",
e9b652861 "#include <chrono>\n",
e9b652862 "#include <chrono>\n#include <cstdint>\n",
e9b652863)
e9b652864replace_exact(
e9b652865 "eden/fs/service/EdenServer.h",
e9b652866 " monitoring::OBCAvg memory_vm_rss_bytes_;\n",
e9b652867 " uint64_t memory_vm_rss_bytes_{0};\n",
e9b652868)
e9b652869replace_exact(
e9b652870 "eden/fs/service/EdenServer.cpp",
e9b652871 """ if (config->enableOBCOnEden.getValue()) {
e9b652872 // Get the hostname without the ".facebook.com" suffix
e9b652873 auto hostname = facebook::network::getLocalHost(/*stripFbDomain=*/true);
e9b652874 std::vector<std::string> entities;
e9b652875 auto reWorkerID = std::getenv("REMOTE_EXECUTION_WORKER");
e9b652876 if (reWorkerID == nullptr) {
e9b652877 entities = {hostname};
e9b652878 } else {
e9b652879 entities = {hostname, fmt::format("{}:{}", hostname, reWorkerID)};
e9b652880 }
e9b652881 memory_vm_rss_bytes_ = monitoring::OBCAvg(
e9b652882 monitoring::OdsCategoryId::ODS_EDEN,
e9b652883 fmt::format("eden.{}", kMemoryVmRssBytes),
e9b652884 entities);
e9b652885 // Report memory usage stats once every 60 seconds
e9b652886 memoryStatsTask_.updateInterval(60s);
e9b652887 }
e9b652888""",
e9b652889 """ if (config->enableOBCOnEden.getValue()) {
e9b652890 // OBC is Meta-internal; preserve memory stats reporting cadence in OSS.
e9b652891 memoryStatsTask_.updateInterval(60s);
e9b652892 }
e9b652893""",
e9b652894)
e9b652895
e9b652896# eden/fs/store/sl: remove OBCPxx header and concrete metric types
e9b652897replace_exact(
e9b652898 "eden/fs/store/sl/SaplingBackingStore.h",
e9b652899 '#include "monitoring/obc/OBCPxx.h"\n',
e9b6528100 "",
e9b6528101)
e9b6528102replace_exact(
e9b6528103 "eden/fs/store/sl/SaplingBackingStore.h",
e9b6528104 "#include <atomic>\n",
e9b6528105 "#include <atomic>\n#include <cstdint>\n",
e9b6528106)
e9b6528107replace_exact(
e9b6528108 "eden/fs/store/sl/SaplingBackingStore.h",
e9b6528109 " monitoring::OBCP99P95P50 getBlobPerRepoLatencies_; // calculates p50, p95, p99\n",
e9b6528110 " uint64_t getBlobPerRepoLatencies_{0}; // OSS fallback counter\n",
e9b6528111)
e9b6528112replace_exact(
e9b6528113 "eden/fs/store/sl/SaplingBackingStore.h",
e9b6528114 " monitoring::OBCP99P95P50 getTreePerRepoLatencies_; // calculates p50, p95, p99\n",
e9b6528115 " uint64_t getTreePerRepoLatencies_{0}; // OSS fallback counter\n",
e9b6528116)
e9b6528117replace_exact(
e9b6528118 "eden/fs/store/sl/SaplingBackingStore.cpp",
e9b6528119 """void SaplingBackingStore::initializeOBCCounters() {
e9b6528120 // Get the hostname without the ".facebook.com" suffix
e9b6528121 auto hostname = facebook::network::getLocalHost(/*stripFbDomain=*/true);
e9b6528122 getBlobPerRepoLatencies_ = monitoring::OBCP99P95P50(
e9b6528123 monitoring::OdsCategoryId::ODS_EDEN,
e9b6528124 fmt::format("eden.store.sapling.fetch_blob_{}_us", repoName_),
e9b6528125 {hostname});
e9b6528126 getTreePerRepoLatencies_ = monitoring::OBCP99P95P50(
e9b6528127 monitoring::OdsCategoryId::ODS_EDEN,
e9b6528128 fmt::format("eden.store.sapling.fetch_tree_{}_us", repoName_),
e9b6528129 {hostname});
e9b6528130 isOBCEnabled_ = true;
e9b6528131}
e9b6528132""",
e9b6528133 """void SaplingBackingStore::initializeOBCCounters() {
e9b6528134 isOBCEnabled_ = true;
e9b6528135}
e9b6528136""",
e9b6528137)
e9b6528138PY
e9b6528139PATCH_OBC
e9b6528140
679ea08141# Additional OSS-only source patching:
679ea08142# - Disable redirect_ffi CXX bridge usage (lib.rs.h is not generated in CMake build)
679ea08143# - Avoid duplicate class definitions from copied backingstore headers
679ea08144RUN <<'PATCH_OSS'
679ea08145python3 - <<'PY'
679ea08146from pathlib import Path
679ea08147
679ea08148
679ea08149def replace_exact(path: str, old: str, new: str) -> None:
679ea08150 p = Path(path)
679ea08151 s = p.read_text()
679ea08152 if old not in s:
679ea08153 raise SystemExit(f"pattern not found in {path}: {old[:80]!r}")
679ea08154 p.write_text(s.replace(old, new))
679ea08155
679ea08156
679ea08157replace_exact(
679ea08158 "eden/fs/service/EdenServiceHandler.cpp",
679ea08159 '#include "eden/fs/rust/redirect_ffi/include/ffi.h"\n',
679ea08160 "",
679ea08161)
679ea08162replace_exact(
679ea08163 "eden/fs/service/EdenServiceHandler.cpp",
679ea08164 '#include "eden/fs/rust/redirect_ffi/src/lib.rs.h"\n',
679ea08165 "",
679ea08166)
679ea08167replace_exact(
679ea08168 "eden/fs/service/EdenServiceHandler.cpp",
679ea08169 """void EdenServiceHandler::listRedirections(
679ea08170 ListRedirectionsResponse& response,
679ea08171 std::unique_ptr<ListRedirectionsRequest> request) {
679ea08172 auto mountId = request->mount();
679ea08173 auto helper = INSTRUMENT_THRIFT_CALL(DBG3, *mountId);
679ea08174
679ea08175 const auto& configDir = server_->getEdenDir();
679ea08176 const auto& edenEtcDir =
679ea08177 server_->getServerState()->getEdenConfig()->getSystemConfigDir();
679ea08178
679ea08179 auto redirsFFI = list_redirections(
679ea08180 absolutePathFromThrift(*mountId->mountPoint()).stringWithoutUNC(),
679ea08181 configDir.stringWithoutUNC(),
679ea08182 edenEtcDir.stringWithoutUNC());
679ea08183
679ea08184 std::vector<Redirection> redirs(redirsFFI.size());
679ea08185 std::transform(
679ea08186 redirsFFI.begin(), redirsFFI.end(), redirs.begin(), [](auto&& redirFFI) {
679ea08187 return redirectionFromFFI(std::move(redirFFI));
679ea08188 });
679ea08189
679ea08190 response.redirections() = std::move(redirs);
679ea08191}
679ea08192""",
679ea08193 """void EdenServiceHandler::listRedirections(
679ea08194 ListRedirectionsResponse& response,
679ea08195 std::unique_ptr<ListRedirectionsRequest> request) {
679ea08196 auto mountId = request->mount();
679ea08197 auto helper = INSTRUMENT_THRIFT_CALL(DBG3, *mountId);
679ea08198 (void)request;
679ea08199 response.redirections() = std::vector<Redirection>{};
679ea08200}
679ea08201""",
679ea08202)
0315de8203replace_exact(
0315de8204 "eden/fs/store/ObjectStore.cpp",
0315de8205 " co_return std::move(result.blob);\n",
0315de8206 " std::shared_ptr<const Blob> blob = result.blob.get();\n"
0315de8207 " co_return blob;\n",
0315de8208)
679ea08209PY
679ea08210PATCH_OSS
679ea08211
e33d237212# Fix ambiguous Cargo.toml deps (both path and git specified — shipit bug)
e33d237213RUN <<'FIX_CARGO'
e33d237214python3 -c "
e33d237215import re, glob
e33d237216for f in glob.glob('eden/fs/cli_rs/*/Cargo.toml'):
e33d237217 with open(f) as fh: c = fh.read()
e33d237218 c = re.sub(r', git = \"[^\"]+\", branch = \"[^\"]+\"', '', c)
e33d237219 with open(f, 'w') as fh: fh.write(c)
e33d237220"
e33d237221FIX_CARGO
e33d237222
4dfd09b223# Create minimal iobuf stub crate — wraps folly::IOBuf for the CXX bridge.
4dfd09b224# The real iobuf crate is Meta-internal; this stub provides just enough for
4dfd09b225# the backingstore FFI to compile. folly::IOBuf is available from the C++ deps.
4dfd09b226RUN mkdir -p eden/scm/lib/iobuf/src
4dfd09b227RUN <<'IOBUF_CARGO'
4dfd09b228cat > eden/scm/lib/iobuf/Cargo.toml << 'INNER'
4dfd09b229[package]
4dfd09b230name = "iobuf"
4dfd09b231version = "0.1.0"
4dfd09b232edition = "2021"
4dfd09b233
4dfd09b234[dependencies]
4dfd09b235cxx = "1.0.119"
4dfd09b236INNER
4dfd09b237IOBUF_CARGO
4dfd09b238RUN <<'IOBUF_RS'
4dfd09b239cat > eden/scm/lib/iobuf/src/lib.rs << 'INNER'
0b1c50f240use std::fmt;
0b1c50f241
e33d237242#[cxx::bridge(namespace = "folly")]
e33d237243mod ffi {
e33d237244 unsafe extern "C++" {
e33d237245 type IOBuf;
e33d237246 }
e33d237247 impl UniquePtr<IOBuf> {}
0b1c50f248}
0b1c50f249
e33d237250pub use ffi::IOBuf;
4dfd09b251
0b1c50f252#[derive(Clone)]
4dfd09b253pub struct IOBufShared {
4dfd09b254 data: Vec<u8>,
4dfd09b255}
4dfd09b256
0b1c50f257impl fmt::Debug for IOBufShared {
0b1c50f258 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
0b1c50f259 f.debug_struct("IOBufShared").field("len", &self.data.len()).finish()
0b1c50f260 }
0b1c50f261}
0b1c50f262
4dfd09b263impl IOBufShared {
4dfd09b264 pub unsafe fn from_owner<T: AsRef<[u8]>>(owner: T) -> Self {
4dfd09b265 Self { data: owner.as_ref().to_vec() }
4dfd09b266 }
4dfd09b267 pub fn append_to_end(&mut self, other: IOBufShared) {
4dfd09b268 self.data.extend_from_slice(&other.data);
4dfd09b269 }
4dfd09b270 pub fn len(&self) -> usize { self.data.len() }
4dfd09b271 pub fn is_empty(&self) -> bool { self.data.is_empty() }
4dfd09b272}
4dfd09b273
4dfd09b274impl From<&str> for IOBufShared {
4dfd09b275 fn from(s: &str) -> Self { Self { data: s.as_bytes().to_vec() } }
4dfd09b276}
4dfd09b277
4dfd09b278impl From<IOBufShared> for Vec<u8> {
4dfd09b279 fn from(buf: IOBufShared) -> Vec<u8> { buf.data }
4dfd09b280}
4dfd09b281
4dfd09b282impl PartialEq for IOBufShared {
4dfd09b283 fn eq(&self, other: &Self) -> bool { self.data == other.data }
4dfd09b284}
4dfd09b285INNER
4dfd09b286IOBUF_RS
0b1c50f287# Add iobuf to workspace + deps, and patch source for OSS compatibility.
0b1c50f288RUN <<'PATCH_SCRIPT'
0b1c50f289#!/bin/bash
0b1c50f290set -e
0b1c50f291
0b1c50f292# Add iobuf to workspace members
0b1c50f293sed -i '/"lib\/identity",/a\ "lib/iobuf",' eden/scm/Cargo.toml
0b1c50f294
0b1c50f295# Add iobuf dep to backingstore
0b1c50f296sed -i '/^cxx = /a iobuf = { version = "0.1.0", path = "../iobuf" }' eden/scm/lib/backingstore/Cargo.toml
0b1c50f297
0b1c50f298# Add iobuf dep to blob
0b1c50f299sed -i '/^sha1 = /a iobuf = { version = "0.1.0", path = "../iobuf" }' eden/scm/lib/blob/Cargo.toml
0b1c50f300
0b1c50f301# Patch ffi.rs: add impl UniquePtr<IOBuf> {}, add make_iobuf_from_bytes fn,
0b1c50f302# replace into_iobuf().into() calls
0b1c50f303python3 -c "
0b1c50f304import re
0b1c50f305with open('eden/scm/lib/backingstore/src/ffi.rs', 'r') as f:
0b1c50f306 content = f.read()
0b1c50f307
e33d237308# Add make_iobuf_from_bytes fn to the extern C++ block containing IOBuf
0b1c50f309content = content.replace(
0b1c50f310 'type IOBuf = iobuf::IOBuf;',
e33d237311 'type IOBuf = iobuf::IOBuf;\n fn make_iobuf_from_bytes(data: &[u8]) -> UniquePtr<IOBuf>;'
0b1c50f312)
0b1c50f313
0b1c50f314# Replace blob.into_iobuf().into() with our helper
0b1c50f315content = content.replace(
0b1c50f316 'blob.into_iobuf().into()',
0b1c50f317 'ffi::make_iobuf_from_bytes(&blob.into_vec())'
0b1c50f318)
0b1c50f319
0b1c50f320with open('eden/scm/lib/backingstore/src/ffi.rs', 'w') as f:
0b1c50f321 f.write(content)
0b1c50f322"
0b1c50f323
0b1c50f324# Add C++ helper to create IOBuf from bytes
0b1c50f325cat >> eden/scm/lib/backingstore/include/ffi.h << 'CPP_EOF'
0b1c50f326
0b1c50f327#include <folly/io/IOBuf.h>
0b1c50f328namespace sapling {
0b1c50f329inline std::unique_ptr<folly::IOBuf> make_iobuf_from_bytes(
0b1c50f330 rust::Slice<const uint8_t> data) {
0b1c50f331 return folly::IOBuf::copyBuffer(data.data(), data.length());
0b1c50f332}
0b1c50f333} // namespace sapling
0b1c50f334CPP_EOF
0b1c50f335PATCH_SCRIPT
4dfd09b336
4dfd09b337# Build EdenFS using getdeps. Unlike Mononoke (pure Cargo), EdenFS uses CMake
2d46ffc338# which needs proper workspace resolution for the Rust FFI crates.
99f1a2e339RUN --mount=type=cache,target=/root/.cargo/registry \
99f1a2e340 python3 build/fbcode_builder/getdeps.py --allow-system-packages \
99f1a2e341 build --no-deps --build-type MinSizeRel --no-tests --src-dir=. eden \
99f1a2e342 --project-install-prefix eden:/
99f1a2e343
99f1a2e344# Collect artifacts with dynamic library fixups
99f1a2e345RUN python3 build/fbcode_builder/getdeps.py --allow-system-packages \
99f1a2e346 fixup-dyn-deps --strip --src-dir=. eden \
99f1a2e347 /artifacts --project-install-prefix eden:/ \
99f1a2e348 --final-install-prefix /usr/local
99f1a2e349
99f1a2e350# =============================================================================
2d46ffc351# Stage 2: Minimal runtime image
99f1a2e352# =============================================================================
99f1a2e353FROM ubuntu:22.04 AS runtime
99f1a2e354
99f1a2e355RUN apt-get update && apt-get install -y \
99f1a2e356 ca-certificates \
99f1a2e357 git \
99f1a2e358 libssl3 \
99f1a2e359 zlib1g \
99f1a2e360 libzstd1 \
99f1a2e361 liblz4-1 \
99f1a2e362 libsnappy1v5 \
99f1a2e363 libsodium23 \
99f1a2e364 libevent-2.1-7 \
99f1a2e365 libdouble-conversion3 \
99f1a2e366 libgflags2.2 \
99f1a2e367 libgoogle-glog0v5 \
99f1a2e368 libunwind8 \
99f1a2e369 libdwarf1 \
99f1a2e370 libfuse2 \
99f1a2e371 libre2-9 \
99f1a2e372 libgit2-1.1 \
99f1a2e373 liblmdb0 \
99f1a2e374 libsqlite3-0 \
99f1a2e375 libcurl4 \
99f1a2e376 python3 \
99f1a2e377 && rm -rf /var/lib/apt/lists/*
99f1a2e378
99f1a2e379# Copy built artifacts (fixup-dyn-deps puts binaries in /artifacts/bin/)
99f1a2e380COPY --from=builder /artifacts/bin /usr/local/bin
99f1a2e381
99f1a2e382# Create data directories
99f1a2e383RUN mkdir -p /data/eden /config /certs
99f1a2e384
99f1a2e385# EdenFS uses FUSE on Linux — needs /dev/fuse access at runtime
99f1a2e386# (requires --privileged or --device /dev/fuse when running the container)
99f1a2e387
99f1a2e388# edenfs daemon port (thrift)
99f1a2e389EXPOSE 8400
99f1a2e390
99f1a2e391# Default entrypoint runs the edenfs daemon
99f1a2e392ENTRYPOINT ["/usr/local/bin/edenfs"]
99f1a2e393CMD ["--edenDir", "/data/eden", \
99f1a2e394 "--etcEdenDir", "/config", \
99f1a2e395 "--configPath", "/config/edenfs.toml", \
99f1a2e396 "--foreground"]