Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor(storage): fix confusing naming #18701

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+493 −480
Open

refactor(storage): fix confusing naming #18701

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
747778e
refactor(storage): fix confusing naming
zwang28 Sep 25, 2024
82bc246
revert proto changes
zwang28 Sep 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/ctl/src/cmd_impl/hummock/list_version.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,15 @@ pub async fn list_version(
{
let l0 = &mut levels.l0;
for sub_level in &mut l0.sub_levels {
for t in &mut sub_level.table_infos {
for t in &mut sub_level.sstable_infos {
t.remove_key_range();
}
}
}

// l1 ~ lmax
for level in &mut levels.levels {
for t in &mut level.table_infos {
for t in &mut level.sstable_infos {
t.remove_key_range();
}
}
Expand All @@ -65,7 +65,7 @@ pub async fn list_version(
"sub_level_id {} type {} sst_num {} size {}",
sub_level.sub_level_id,
sub_level.level_type.as_str_name(),
sub_level.table_infos.len(),
sub_level.sstable_infos.len(),
sub_level.total_file_size
)
}
Expand All @@ -76,7 +76,7 @@ pub async fn list_version(
"level_idx {} type {} sst_num {} size {}",
level.level_idx,
level.level_type.as_str_name(),
level.table_infos.len(),
level.sstable_infos.len(),
level.total_file_size
)
}
Expand Down
2 changes: 1 addition & 1 deletion src/ctl/src/cmd_impl/hummock/sst_dump.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ pub async fn sst_dump(context: &CtlContext, args: SstDumpArgs) -> anyhow::Result
let version = hummock.inner().get_pinned_version().version().clone();
let sstable_store = hummock.sstable_store();
for level in version.get_combined_levels() {
for sstable_info in &level.table_infos {
for sstable_info in &level.sstable_infos {
if let Some(object_id) = &args.object_id {
if *object_id == sstable_info.object_id {
print_level(level, sstable_info);
Expand Down
2 changes: 1 addition & 1 deletion src/ctl/src/cmd_impl/hummock/validate_version.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ async fn print_user_key_in_version(
println!("print key {:?} in version {}", target_key, version.id);
for cg in version.levels.values() {
for level in cg.l0.sub_levels.iter().rev().chain(cg.levels.iter()) {
for sstable_info in &level.table_infos {
for sstable_info in &level.sstable_infos {
let key_range = &sstable_info.key_range;
let left_user_key = FullKey::decode(&key_range.left);
let right_user_key = FullKey::decode(&key_range.right);
Expand Down
4 changes: 2 additions & 2 deletions src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ fn remove_key_range_from_version(mut version: HummockVersion) -> HummockVersion
// Because key range is too verbose for manual analysis, just don't expose it.
for cg in version.levels.values_mut() {
for level in cg.levels.iter_mut().chain(cg.l0.sub_levels.iter_mut()) {
for sst in &mut level.table_infos {
for sst in &mut level.sstable_infos {
sst.remove_key_range();
}
}
Expand All @@ -111,7 +111,7 @@ fn version_to_sstable_rows(version: HummockVersion) -> Vec<RwHummockSstable> {
let mut sstables = vec![];
for cg in version.levels.into_values() {
for level in cg.levels.into_iter().chain(cg.l0.sub_levels) {
for sst in level.table_infos {
for sst in level.sstable_infos {
let key_range = sst.key_range;
sstables.push(RwHummockSstable {
sstable_id: sst.sst_id as _,
Expand Down
4 changes: 2 additions & 2 deletions src/meta/src/hummock/compaction/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ impl CompactStatus {
}

if task.input_ssts[1].level_idx == task.target_level
&& task.input_ssts[1].table_infos.is_empty()
&& task.input_ssts[1].sstable_infos.is_empty()
{
return true;
}
Expand All @@ -178,7 +178,7 @@ impl CompactStatus {
}
let exist_table_ids = HashSet::<u32>::from_iter(task.existing_table_ids.clone());
task.input_ssts.iter().all(|level| {
level.table_infos.iter().all(|sst| {
level.sstable_infos.iter().all(|sst| {
sst.table_ids
.iter()
.all(|table_id| !exist_table_ids.contains(table_id))
Expand Down
72 changes: 36 additions & 36 deletions src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ impl CompactionPicker for LevelCompactionPicker {
return None;
}
if l0.sub_levels[0].level_type != LevelType::Nonoverlapping
&& l0.sub_levels[0].table_infos.len() > 1
&& l0.sub_levels[0].sstable_infos.len() > 1
{
stats.skip_by_overlapping += 1;
return None;
Expand Down Expand Up @@ -145,8 +145,8 @@ impl LevelCompactionPicker {
);

trivial_move_picker.pick_trivial_move_task(
&l0.sub_levels[0].table_infos,
&target_level.table_infos,
&l0.sub_levels[0].sstable_infos,
&target_level.sstable_infos,
level_handlers,
stats,
)
Expand Down Expand Up @@ -208,7 +208,7 @@ impl LevelCompactionPicker {
.collect_vec();

let target_level_ssts = overlap_strategy
.check_base_level_overlap(&l0_select_tables, &target_level.table_infos);
.check_base_level_overlap(&l0_select_tables, &target_level.sstable_infos);

let mut target_level_size = 0;
let mut pending_compact = false;
Expand Down Expand Up @@ -243,15 +243,15 @@ impl LevelCompactionPicker {
.map(|table_infos| InputLevel {
level_idx: 0,
level_type: LevelType::Nonoverlapping,
table_infos,
sstable_infos: table_infos,
})
.collect_vec();
select_level_inputs.reverse();
let target_file_count = target_level_files.len();
select_level_inputs.push(InputLevel {
level_idx: target_level.level_idx,
level_type: target_level.level_type,
table_infos: target_level_files,
sstable_infos: target_level_files,
});

let result = CompactionInput {
Expand Down Expand Up @@ -345,9 +345,9 @@ pub mod tests {
let ret = picker
.pick_compaction(&levels, &levels_handler, &mut local_stats)
.unwrap();
assert_eq!(ret.input_levels[0].table_infos.len(), 1);
assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 4);
assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 1);
assert_eq!(ret.input_levels[0].sstable_infos.len(), 1);
assert_eq!(ret.input_levels[0].sstable_infos[0].sst_id, 4);
assert_eq!(ret.input_levels[1].sstable_infos[0].sst_id, 1);

ret.add_pending_task(0, &mut levels_handler);
{
Expand All @@ -357,15 +357,15 @@ pub mod tests {
.pick_compaction(&levels, &levels_handler, &mut local_stats)
.unwrap();

assert_eq!(ret2.input_levels[0].table_infos.len(), 1);
assert_eq!(ret2.input_levels[0].table_infos[0].sst_id, 6);
assert_eq!(ret2.input_levels[1].table_infos[0].sst_id, 5);
assert_eq!(ret2.input_levels[0].sstable_infos.len(), 1);
assert_eq!(ret2.input_levels[0].sstable_infos[0].sst_id, 6);
assert_eq!(ret2.input_levels[1].sstable_infos[0].sst_id, 5);
}

levels.l0.sub_levels[0]
.table_infos
.sstable_infos
.retain(|table| table.sst_id != 4);
levels.l0.total_file_size -= ret.input_levels[0].table_infos[0].file_size;
levels.l0.total_file_size -= ret.input_levels[0].sstable_infos[0].file_size;

levels_handler[0].remove_task(0);
levels_handler[1].remove_task(0);
Expand All @@ -374,11 +374,11 @@ pub mod tests {
.pick_compaction(&levels, &levels_handler, &mut local_stats)
.unwrap();
assert_eq!(ret.input_levels.len(), 3);
assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 6);
assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 5);
assert_eq!(ret.input_levels[2].table_infos.len(), 2);
assert_eq!(ret.input_levels[2].table_infos[0].sst_id, 3);
assert_eq!(ret.input_levels[2].table_infos[1].sst_id, 2);
assert_eq!(ret.input_levels[0].sstable_infos[0].sst_id, 6);
assert_eq!(ret.input_levels[1].sstable_infos[0].sst_id, 5);
assert_eq!(ret.input_levels[2].sstable_infos.len(), 2);
assert_eq!(ret.input_levels[2].sstable_infos[0].sst_id, 3);
assert_eq!(ret.input_levels[2].sstable_infos[1].sst_id, 2);
ret.add_pending_task(1, &mut levels_handler);

let mut local_stats = LocalPickerStatistic::default();
Expand All @@ -395,9 +395,9 @@ pub mod tests {
.pick_compaction(&levels, &levels_handler, &mut local_stats)
.unwrap();
assert_eq!(ret.input_levels.len(), 3);
assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 6);
assert_eq!(ret.input_levels[1].table_infos[0].sst_id, 5);
assert_eq!(ret.input_levels[2].table_infos.len(), 2);
assert_eq!(ret.input_levels[0].sstable_infos[0].sst_id, 6);
assert_eq!(ret.input_levels[1].sstable_infos[0].sst_id, 5);
assert_eq!(ret.input_levels[2].sstable_infos.len(), 2);
}
#[test]
fn test_selecting_key_range_overlap() {
Expand All @@ -415,7 +415,7 @@ pub mod tests {
let levels = vec![Level {
level_idx: 1,
level_type: LevelType::Nonoverlapping,
table_infos: vec![
sstable_infos: vec![
generate_table(3, 1, 0, 50, 1),
generate_table(4, 1, 150, 200, 1),
generate_table(5, 1, 250, 300, 1),
Expand Down Expand Up @@ -453,7 +453,7 @@ pub mod tests {
assert_eq!(ret.input_levels.len(), 2);
assert_eq!(
ret.input_levels[0]
.table_infos
.sstable_infos
.iter()
.map(|t| t.sst_id)
.collect_vec(),
Expand All @@ -462,7 +462,7 @@ pub mod tests {

assert_eq!(
ret.input_levels[1]
.table_infos
.sstable_infos
.iter()
.map(|t| t.sst_id)
.collect_vec(),
Expand All @@ -478,7 +478,7 @@ pub mod tests {
let levels = vec![Level {
level_idx: 1,
level_type: LevelType::Nonoverlapping,
table_infos: vec![],
sstable_infos: vec![],
total_file_size: 0,
sub_level_id: 0,
uncompressed_file_size: 0,
Expand Down Expand Up @@ -542,7 +542,7 @@ pub mod tests {
levels: vec![Level {
level_idx: 1,
level_type: LevelType::Nonoverlapping,
table_infos: vec![
sstable_infos: vec![
generate_table(1, 1, 100, 399, 2),
generate_table(2, 1, 400, 699, 2),
generate_table(3, 1, 700, 999, 2),
Expand All @@ -566,7 +566,7 @@ pub mod tests {

let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
let mut local_stats = LocalPickerStatistic::default();
levels_handler[0].add_pending_task(1, 4, &levels.l0.sub_levels[0].table_infos);
levels_handler[0].add_pending_task(1, 4, &levels.l0.sub_levels[0].sstable_infos);
let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
// Skip this compaction because the write amplification is too large.
assert!(ret.is_none());
Expand Down Expand Up @@ -610,7 +610,7 @@ pub mod tests {
let ret = picker
.pick_compaction(&levels, &levels_handler, &mut local_stats)
.unwrap();
assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 7);
assert_eq!(ret.input_levels[0].sstable_infos[0].sst_id, 7);
assert_eq!(
3,
ret.input_levels.iter().filter(|l| l.level_idx == 0).count()
Expand All @@ -620,7 +620,7 @@ pub mod tests {
ret.input_levels
.iter()
.filter(|l| l.level_idx == 0)
.map(|l| l.table_infos.len())
.map(|l| l.sstable_infos.len())
.sum::<usize>()
);

Expand All @@ -638,7 +638,7 @@ pub mod tests {
let ret = picker
.pick_compaction(&levels, &levels_handler, &mut local_stats)
.unwrap();
assert_eq!(ret.input_levels[0].table_infos[0].sst_id, 6);
assert_eq!(ret.input_levels[0].sstable_infos[0].sst_id, 6);
assert_eq!(
2,
ret.input_levels.iter().filter(|l| l.level_idx == 0).count()
Expand All @@ -648,7 +648,7 @@ pub mod tests {
ret.input_levels
.iter()
.filter(|l| l.level_idx == 0)
.map(|l| l.table_infos.len())
.map(|l| l.sstable_infos.len())
.sum::<usize>()
);
}
Expand Down Expand Up @@ -679,7 +679,7 @@ pub mod tests {
input_levels: vec![InputLevel {
level_idx: 0,
level_type: pending_level.level_type,
table_infos: pending_level.table_infos.clone(),
sstable_infos: pending_level.sstable_infos.clone(),
}],
target_level: 1,
target_sub_level_id: pending_level.sub_level_id,
Expand Down Expand Up @@ -753,14 +753,14 @@ pub mod tests {
.unwrap();
// 1. trivial_move
assert_eq!(2, ret.input_levels.len());
assert!(ret.input_levels[1].table_infos.is_empty());
assert_eq!(5, ret.input_levels[0].table_infos[0].sst_id);
assert!(ret.input_levels[1].sstable_infos.is_empty());
assert_eq!(5, ret.input_levels[0].sstable_infos[0].sst_id);
ret.add_pending_task(0, &mut levels_handler);

let ret = picker
.pick_compaction(&levels, &levels_handler, &mut local_stats)
.unwrap();
assert_eq!(3, ret.input_levels.len());
assert_eq!(6, ret.input_levels[0].table_infos[0].sst_id);
assert_eq!(6, ret.input_levels[0].sstable_infos[0].sst_id);
}
}
2 changes: 1 addition & 1 deletion src/meta/src/hummock/compaction/picker/compaction_task_validator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ impl CompactionTaskValidationRule for IntraCompactionTaskValidationRule {
let mut max_level_size = 0;
for select_level in &input.input_levels {
let level_select_size = select_level
.table_infos
.sstable_infos
.iter()
.map(|sst| sst.sst_size)
.sum::<u64>();
Expand Down
Loading
Loading