Skip to content

Commit 36d9c76

Browse files
markbtfacebook-github-bot
authored andcommitted
hooks: add hook for limiting subtree copy source sizes
Summary: Add a new hook that lets us limit the size of subtree copy sources, and the depth of the destination. There are two aspects of "size" we care about when it comes to limiting subtree operations: the size of the source, and the size of the covered portion of the repository at the destination. For source size we limit the total number of files located at the source. For the destination, we require that it has a minimum path depth. Reviewed By: andreacampi Differential Revision: D71842445 fbshipit-source-id: c89713185eb56975b48e9af91a6176817f870274
1 parent b31d010 commit 36d9c76

File tree

5 files changed

+240
-3
lines changed

5 files changed

+240
-3
lines changed

eden/mononoke/hooks/BUCK

+4-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ rust_library(
2020
"//common/rust/shed/borrowed:borrowed",
2121
"//common/rust/shed/fbinit:fbinit-tokio",
2222
"//eden/mononoke/blobrepo:repo_blobstore",
23-
"//eden/mononoke/blobstore:blobstore",
2423
"//eden/mononoke/mononoke_macros:mononoke_macros",
2524
"//eden/mononoke/mononoke_types:mononoke_types",
2625
"//eden/mononoke/mononoke_types:mononoke_types-mocks",
@@ -56,9 +55,12 @@ rust_library(
5655
"//configerator/structs/infrasec/if:acl-rust",
5756
"//crypto/keychain_service:if-rust",
5857
"//crypto/keychain_service:if-rust-srclients",
58+
"//eden/mononoke/blobrepo:repo_blobstore",
59+
"//eden/mononoke/blobstore:blobstore",
5960
"//eden/mononoke/bookmarks:bookmarks",
6061
"//eden/mononoke/common/facebook/thrift_client:thrift_client",
6162
"//eden/mononoke/common/scuba_ext:scuba_ext",
63+
"//eden/mononoke/derived_data:skeleton_manifest",
6264
"//eden/mononoke/facebook/employee_service:employee_service",
6365
"//eden/mononoke/facebook/phabricator:phabricator",
6466
"//eden/mononoke/manifest:manifest",
@@ -68,6 +70,7 @@ rust_library(
6870
"//eden/mononoke/permission_checker:permission_checker",
6971
"//eden/mononoke/repo_attributes/hook_manager/hook_manager:hook_manager",
7072
"//eden/mononoke/repo_attributes/repo_cross_repo:repo_cross_repo",
73+
"//eden/mononoke/repo_attributes/repo_derived_data:repo_derived_data",
7174
"//eden/mononoke/repo_attributes/repo_identity:repo_identity",
7275
"//eden/mononoke/scs/if:source_control-rust",
7376
"//eden/mononoke/scs/if:source_control-rust-clients",

eden/mononoke/hooks/Cargo.toml

+4-2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ doctest = false
1414
[dependencies]
1515
anyhow = "1.0.95"
1616
async-trait = "0.1.86"
17+
blobstore = { version = "0.1.0", path = "../blobstore" }
1718
bookmarks = { version = "0.1.0", path = "../bookmarks" }
1819
bytes = { version = "1.9.0", features = ["serde"] }
1920
context = { version = "0.1.0", path = "../server/context" }
@@ -34,14 +35,17 @@ metaconfig_types = { version = "0.1.0", path = "../metaconfig/types" }
3435
mononoke_types = { version = "0.1.0", path = "../mononoke_types" }
3536
permission_checker = { version = "0.1.0", path = "../permission_checker" }
3637
regex = "1.11.1"
38+
repo_blobstore = { version = "0.1.0", path = "../blobrepo/repo_blobstore" }
3739
repo_cross_repo = { version = "0.1.0", path = "../repo_attributes/repo_cross_repo" }
40+
repo_derived_data = { version = "0.1.0", path = "../repo_attributes/repo_derived_data" }
3841
repo_identity = { version = "0.1.0", path = "../repo_attributes/repo_identity" }
3942
sapling-clientinfo = { version = "0.1.0", path = "../../scm/lib/clientinfo" }
4043
scuba_ext = { version = "0.1.0", path = "../common/scuba_ext" }
4144
serde = { version = "1.0.185", features = ["derive", "rc"] }
4245
serde_derive = "1.0.185"
4346
serde_json = { version = "1.0.140", features = ["float_roundtrip", "unbounded_depth"] }
4447
serde_regex = "1"
48+
skeleton_manifest = { version = "0.1.0", path = "../derived_data/skeleton_manifest" }
4549
slog = { package = "tracing_slog_compat", version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
4650
source_control = { version = "0.1.0", path = "../scs/if" }
4751
source_control_clients = { version = "0.1.0", path = "../scs/if/clients" }
@@ -50,15 +54,13 @@ thiserror = "2"
5054
tokio = { version = "1.41.0", features = ["full", "test-util", "tracing"] }
5155

5256
[dev-dependencies]
53-
blobstore = { version = "0.1.0", path = "../blobstore" }
5457
borrowed = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
5558
fbinit-tokio = { version = "0.1.2", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
5659
fixtures = { version = "0.1.0", path = "../tests/fixtures" }
5760
hook_manager_testlib = { version = "0.1.0", path = "../repo_attributes/hook_manager/hook_manager_testlib" }
5861
mononoke_macros = { version = "0.1.0", path = "../mononoke_macros" }
5962
mononoke_types-mocks = { version = "0.1.0", path = "../mononoke_types/mocks" }
6063
pretty_assertions = { version = "1.2", features = ["alloc"], default-features = false }
61-
repo_blobstore = { version = "0.1.0", path = "../blobrepo/repo_blobstore" }
6264
repo_permission_checker = { version = "0.1.0", path = "../repo_attributes/repo_permission_checker" }
6365
test_repo_factory = { version = "0.1.0", path = "../repo_factory/test_repo_factory" }
6466
tests_utils = { version = "0.1.0", path = "../tests/utils" }

eden/mononoke/hooks/src/implementations.rs

+4
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ mod limit_directory_size;
2626
pub(crate) mod limit_filesize;
2727
mod limit_path_length;
2828
pub(crate) mod limit_submodule_edits;
29+
mod limit_subtree_op_size;
2930
pub(crate) mod limit_tag_updates;
3031
pub(crate) mod missing_lfsconfig;
3132
pub(crate) mod no_bad_filenames;
@@ -104,6 +105,9 @@ pub async fn make_changeset_hook(
104105
"limit_submodule_edits" => Some(b(limit_submodule_edits::LimitSubmoduleEditsHook::new(
105106
&params.config,
106107
)?)),
108+
"limit_subtree_op_size" => Some(b(limit_subtree_op_size::LimitSubtreeOpSizeHook::new(
109+
&params.config,
110+
)?)),
107111
"missing_lfsconfig" => Some(b(missing_lfsconfig::MissingLFSConfigHook::new())),
108112
"block_commit_message_pattern" => Some(b(
109113
block_commit_message_pattern::BlockCommitMessagePatternHook::new(&params.config)?,
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* This software may be used and distributed according to the terms of the
5+
* GNU General Public License version 2.
6+
*/
7+
8+
use anyhow::Result;
9+
use async_trait::async_trait;
10+
use blobstore::StoreLoadable;
11+
use bookmarks::BookmarkKey;
12+
use context::CoreContext;
13+
use hook_manager::ChangesetHook;
14+
use hook_manager::HookRejectionInfo;
15+
use manifest::ManifestOps;
16+
use mononoke_types::BonsaiChangeset;
17+
use repo_blobstore::RepoBlobstoreRef;
18+
use repo_derived_data::RepoDerivedDataRef;
19+
use serde::Deserialize;
20+
use skeleton_manifest::RootSkeletonManifestId;
21+
22+
use crate::CrossRepoPushSource;
23+
use crate::HookConfig;
24+
use crate::HookExecution;
25+
use crate::HookRepo;
26+
use crate::PushAuthoredBy;
27+
28+
#[derive(Deserialize, Clone, Debug)]
29+
pub struct LimitSubtreeOpSizeConfig {
30+
#[serde(default)]
31+
source_file_count_limit: Option<u64>,
32+
33+
#[serde(default)]
34+
dest_min_path_depth: Option<usize>,
35+
36+
/// Message to use when rejecting a commit because the source directory is too large.
37+
///
38+
/// The following variables are available:
39+
/// - ${dest_path}: The path of the destination.
40+
/// - ${source}: The changeset id of the source.
41+
/// - ${source_path}: The path of the source.
42+
/// - ${count}: The number of files in the source.
43+
/// - ${limit}: The limit on the number of files in the source.
44+
too_many_files_rejection_message: String,
45+
46+
/// Message to use when rejecting a commit because the destination path is too shallow.
47+
///
48+
/// The following variables are available:
49+
/// - ${dest_path}: The path of the destination.
50+
/// - ${dest_min_path_depth}: The minimum depth of the destination path.
51+
dest_too_shallow_rejection_message: String,
52+
}
53+
54+
/// Hook to limit the size of subtree operations.
55+
pub struct LimitSubtreeOpSizeHook {
56+
config: LimitSubtreeOpSizeConfig,
57+
}
58+
59+
impl LimitSubtreeOpSizeHook {
60+
pub fn new(config: &HookConfig) -> Result<Self> {
61+
Self::with_config(config.parse_options()?)
62+
}
63+
64+
pub fn with_config(config: LimitSubtreeOpSizeConfig) -> Result<Self> {
65+
Ok(Self { config })
66+
}
67+
}
68+
69+
#[async_trait]
70+
impl ChangesetHook for LimitSubtreeOpSizeHook {
71+
async fn run<'this: 'cs, 'ctx: 'this, 'cs, 'repo: 'cs>(
72+
&'this self,
73+
ctx: &'ctx CoreContext,
74+
hook_repo: &'repo HookRepo,
75+
_bookmark: &BookmarkKey,
76+
changeset: &'cs BonsaiChangeset,
77+
cross_repo_push_source: CrossRepoPushSource,
78+
push_authored_by: PushAuthoredBy,
79+
) -> Result<HookExecution> {
80+
if push_authored_by.service() {
81+
return Ok(HookExecution::Accepted);
82+
}
83+
if cross_repo_push_source == CrossRepoPushSource::PushRedirected {
84+
// For push-redirected commits, we rely on running source-repo hooks
85+
return Ok(HookExecution::Accepted);
86+
}
87+
88+
for (path, change) in changeset.subtree_changes() {
89+
if let Some(source_file_count_limit) = self.config.source_file_count_limit {
90+
if let Some((source, source_path)) = change.change_source() {
91+
let source_root = hook_repo
92+
.repo_derived_data()
93+
.derive::<RootSkeletonManifestId>(ctx, source)
94+
.await?
95+
.into_skeleton_manifest_id();
96+
let source_entry = source_root
97+
.find_entry(
98+
ctx.clone(),
99+
hook_repo.repo_blobstore().clone(),
100+
source_path.clone(),
101+
)
102+
.await?
103+
.ok_or_else(|| {
104+
anyhow::anyhow!(concat!(
105+
"Source directory {source_path} ",
106+
"not found in source changeset {source}"
107+
))
108+
})?;
109+
let source_count = match source_entry {
110+
manifest::Entry::Tree(tree_id) => {
111+
let source_tree = tree_id.load(ctx, hook_repo.repo_blobstore()).await?;
112+
source_tree.summary().descendant_files_count
113+
}
114+
manifest::Entry::Leaf(_) => 1,
115+
};
116+
117+
if source_count > source_file_count_limit {
118+
return Ok(HookExecution::Rejected(HookRejectionInfo::new_long(
119+
"Subtree source is too large",
120+
self.config
121+
.too_many_files_rejection_message
122+
.replace("${dest_path}", &path.to_string())
123+
.replace("${source}", &source.to_string())
124+
.replace("${source_path}", &source_path.to_string())
125+
.replace("${count}", &source_count.to_string())
126+
.replace("${limit}", &source_file_count_limit.to_string()),
127+
)));
128+
}
129+
}
130+
}
131+
132+
if let Some(dest_min_path_depth) = self.config.dest_min_path_depth {
133+
if path.num_components() < dest_min_path_depth {
134+
return Ok(HookExecution::Rejected(HookRejectionInfo::new_long(
135+
"Subtree destination is too shallow",
136+
self.config
137+
.dest_too_shallow_rejection_message
138+
.replace("${dest_path}", &path.to_string())
139+
.replace("${dest_min_path_depth}", &dest_min_path_depth.to_string()),
140+
)));
141+
}
142+
}
143+
}
144+
145+
Ok(HookExecution::Accepted)
146+
}
147+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
#
3+
# This software may be used and distributed according to the terms of the
4+
# GNU General Public License found in the LICENSE file in the root
5+
# directory of this source tree.
6+
7+
$ . "${TEST_FIXTURES}/library.sh"
8+
9+
Override subtree key to enable non-test subtree extra
10+
$ cat > $TESTTMP/subtree.py <<EOF
11+
> from sapling.utils import subtreeutil
12+
> def extsetup(ui):
13+
> subtreeutil.SUBTREE_KEY = "subtree"
14+
> EOF
15+
$ setconfig extensions.subtreetestoverride=$TESTTMP/subtree.py
16+
$ setconfig push.edenapi=true
17+
$ setconfig subtree.min-path-depth=1
18+
$ enable amend
19+
$ setup_common_config
20+
$ register_hooks \
21+
> limit_subtree_op_size <(
22+
> cat <<CONF
23+
> bypass_pushvar="TEST_BYPASS=true"
24+
> config_json='''{
25+
> "source_file_count_limit": 2,
26+
> "dest_min_path_depth": 2,
27+
> "too_many_files_rejection_message": "Too many files in subtree operation copying from \${source_path} to \${dest_path}: \${count} > \${limit}",
28+
> "dest_too_shallow_rejection_message": "Subtree operation copying to \${dest_path} has too shallow destination path: < \${dest_min_path_depth}"
29+
> }'''
30+
> CONF
31+
> )
32+
33+
$ testtool_drawdag -R repo --derive-all --no-default-files << EOF
34+
> A-B-C-D
35+
> # modify: A foo/file1 "aaa\n"
36+
> # modify: A foo/file3 "xxx\n"
37+
> # copy: B foo/file2 "bbb\n" A foo/file1
38+
> # delete: B foo/file1
39+
> # modify: C foo/file2 "ccc\n"
40+
> # modify: D foo/file4 "yyy\n"
41+
> # bookmark: D master_bookmark
42+
> EOF
43+
A=bad79679db57d8ca7bdcb80d082d1508f33ca2989652922e2e01b55fb3c27f6a
44+
B=170dbba760afb7ec239d859e2412a827dd7229cdbdfcd549b7138b2451afad37
45+
C=e611f471e1f2bd488fee752800983cdbfd38d50247e5d81222e0d620fd2a6120
46+
D=ec1da6035c39c33e159c4baa6b9bfd54676d38a66255ccbcd436f6cfa8ecc2eb
47+
48+
$ start_and_wait_for_mononoke_server
49+
$ hg clone -q mono:repo repo
50+
$ cd repo
51+
52+
$ hg update -q master_bookmark^
53+
$ hg subtree copy -r .^ --from-path foo --to-path bar
54+
copying foo to bar
55+
$ ls bar
56+
file2
57+
file3
58+
$ cat bar/file2
59+
bbb
60+
61+
$ hg push -q -r . --to master_bookmark
62+
abort: Server error: hooks failed:
63+
limit_subtree_op_size for 44bba4bd23a81344c3cbc7f28b42363d4df03399ce6cb0851b51babb48c20549: Subtree operation copying to bar has too shallow destination path: < 2
64+
[255]
65+
66+
$ hg update -q master_bookmark
67+
$ hg subtree copy -r . --from-path foo --to-path bar/baz
68+
copying foo to bar/baz
69+
$ hg push -q -r . --to master_bookmark
70+
abort: Server error: hooks failed:
71+
limit_subtree_op_size for c5c46cbfe6ecfb38810f51d2d0c3532c38f90d02d483f99d967e14add2be71ff: Too many files in subtree operation copying from foo to bar/baz: 3 > 2
72+
[255]
73+
74+
$ hg update -q master_bookmark^
75+
$ hg subtree copy -r . --from-path foo --to-path bar/baz
76+
copying foo to bar/baz
77+
$ hg push -q -r . --to master_bookmark
78+
$ hg update -q master_bookmark
79+
$ ls bar/baz
80+
file2
81+
file3

0 commit comments

Comments
 (0)