From 5c0c1f96d4bd6fd733cdde63d16ebe7851375260 Mon Sep 17 00:00:00 2001 From: lemorage Date: Tue, 6 May 2025 09:03:08 +0200 Subject: [PATCH 1/8] feat: implement flow spec getter in pyo3 bindings --- src/base/spec.rs | 216 +++++++++++++++++++++++++++++++++++++++++++---- src/py/mod.rs | 55 +++++++++++- 2 files changed, 253 insertions(+), 18 deletions(-) diff --git a/src/base/spec.rs b/src/base/spec.rs index 0b61f0b2..eaa2da97 100644 --- a/src/base/spec.rs +++ b/src/base/spec.rs @@ -1,6 +1,8 @@ use crate::prelude::*; -use super::schema::{EnrichedValueType, FieldSchema}; +use super::schema::{EnrichedValueType, FieldSchema, ValueType}; +use serde::{Deserialize, Serialize}; +use std::fmt; use std::ops::Deref; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -34,8 +36,8 @@ impl Deref for FieldPath { } } -impl std::fmt::Display for FieldPath { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for FieldPath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if self.is_empty() { write!(f, "*") } else { @@ -49,8 +51,8 @@ impl std::fmt::Display for FieldPath { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] pub struct OpArgName(pub Option); -impl std::fmt::Display for OpArgName { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for OpArgName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if let Some(arg_name) = &self.0 { write!(f, "${}", arg_name) } else { @@ -73,6 +75,12 @@ pub struct NamedSpec { pub spec: T, } +impl fmt::Display for NamedSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}: {}", self.name, self.spec) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FieldMapping { /// If unspecified, means the current scope. @@ -83,12 +91,36 @@ pub struct FieldMapping { pub field_path: FieldPath, } +impl fmt::Display for FieldMapping { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let scope = self.scope.as_deref().unwrap_or(""); + write!( + f, + "{}{}", + if scope.is_empty() { + "".to_string() + } else { + format!("{}.", scope) + }, + self.field_path + ) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ConstantMapping { pub schema: EnrichedValueType, pub value: serde_json::Value, } +impl fmt::Display for ConstantMapping { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let schema = format_value_type(&self.schema); + let value = serde_json::to_string(&self.value).unwrap_or("#serde_error".to_string()); + write!(f, "Constant({}: {})", value, schema) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CollectionMapping { pub field: FieldMapping, @@ -100,6 +132,18 @@ pub struct StructMapping { pub fields: Vec>, } +impl fmt::Display for StructMapping { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let fields = self + .fields + .iter() + .map(|field| format!("{}={}", field.name, field.spec)) + .collect::>() + .join(", "); + write!(f, "[{}]", fields) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "kind")] pub enum ValueMapping { @@ -155,6 +199,16 @@ pub struct OpArgBinding { pub value: ValueMapping, } +impl fmt::Display for OpArgBinding { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.arg_name.is_unnamed() { + write!(f, "{}", self.value) + } else { + write!(f, "{}={}", self.arg_name, self.value) + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct OpSpec { pub kind: String, @@ -162,11 +216,41 @@ pub struct OpSpec { pub spec: serde_json::Map, } +impl fmt::Display for OpSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let spec_str = serde_json::to_string_pretty(&self.spec) + .map(|s| { + let lines: Vec<&str> = s.lines().take(50).collect(); + if lines.len() < s.lines().count() { + lines + .into_iter() + .chain(["..."]) + .collect::>() + .join("\n ") + } else { + lines.join("\n ") + } + }) + .unwrap_or("#serde_error".to_string()); + write!(f, "OpSpec: kind={}, spec={}", self.kind, spec_str) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct SourceRefreshOptions { pub refresh_interval: Option, } +impl fmt::Display for SourceRefreshOptions { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let refresh = self + .refresh_interval + .map(|d| format!("{:?}", d)) + .unwrap_or("None".to_string()); + write!(f, "{}", refresh) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ImportOpSpec { pub source: OpSpec, @@ -175,14 +259,34 @@ pub struct ImportOpSpec { pub refresh_options: SourceRefreshOptions, } -/// Transform data using a given operator. +impl fmt::Display for ImportOpSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "Import: source={}, refresh={}", + self.source, self.refresh_options + ) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TransformOpSpec { pub inputs: Vec, pub op: OpSpec, } -/// Apply reactive operations to each row of the input field. +impl fmt::Display for TransformOpSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let inputs = self + .inputs + .iter() + .map(|input| input.to_string()) + .collect::>() + .join(", "); + write!(f, "Transform: op={}, inputs=[{}]", self.op, inputs) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ForEachOpSpec { /// Mapping that provides a table to apply reactive operations to. @@ -190,7 +294,12 @@ pub struct ForEachOpSpec { pub op_scope: ReactiveOpScope, } -/// Emit data to a given collector at the given scope. +impl fmt::Display for ForEachOpSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ForEach: field={}", self.field_path) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CollectOpSpec { /// Field values to be collected. @@ -204,6 +313,16 @@ pub struct CollectOpSpec { pub auto_uuid_field: Option, } +impl fmt::Display for CollectOpSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "Collect: scope={}, collector={}, input={}, uuid_field={:?}", + self.scope_name, self.collector_name, self.input, self.auto_uuid_field + ) + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] pub enum VectorSimilarityMetric { CosineSimilarity, @@ -211,8 +330,8 @@ pub enum VectorSimilarityMetric { InnerProduct, } -impl std::fmt::Display for VectorSimilarityMetric { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for VectorSimilarityMetric { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { VectorSimilarityMetric::CosineSimilarity => write!(f, "Cosine"), VectorSimilarityMetric::L2Distance => write!(f, "L2"), @@ -227,6 +346,12 @@ pub struct VectorIndexDef { pub metric: VectorSimilarityMetric, } +impl fmt::Display for VectorIndexDef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:{}", self.field_name, self.metric) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct IndexOptions { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -235,7 +360,27 @@ pub struct IndexOptions { pub vector_indexes: Vec, } -/// Store data to a given sink. +impl fmt::Display for IndexOptions { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let primary_keys = self + .primary_key_fields + .as_ref() + .map(|p| p.join(", ")) + .unwrap_or_default(); + let vector_indexes = self + .vector_indexes + .iter() + .map(|v| v.to_string()) + .collect::>() + .join(", "); + write!( + f, + "IndexOptions: primary_keys=[{}], vector_indexes=[{}]", + primary_keys, vector_indexes + ) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ExportOpSpec { pub collector_name: FieldName, @@ -244,7 +389,16 @@ pub struct ExportOpSpec { pub setup_by_user: bool, } -/// A reactive operation reacts on given input values. +impl fmt::Display for ExportOpSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "Export: collector={}, target={}, {}, setup_by_user={}", + self.collector_name, self.target, self.index_options, self.setup_by_user + ) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "action")] pub enum ReactiveOpSpec { @@ -253,6 +407,16 @@ pub enum ReactiveOpSpec { Collect(CollectOpSpec), } +impl fmt::Display for ReactiveOpSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ReactiveOpSpec::Transform(t) => write!(f, "{}", t), + ReactiveOpSpec::ForEach(fe) => write!(f, "{}", fe), + ReactiveOpSpec::Collect(c) => write!(f, "{}", c), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReactiveOpScope { pub name: ScopeName, @@ -260,7 +424,12 @@ pub struct ReactiveOpScope { // TODO: Suport collectors } -/// A flow defines the rule to sync data from given sources to given sinks with given transformations. +impl fmt::Display for ReactiveOpScope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Scope: name={}", self.name) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FlowInstanceSpec { /// Name of the flow instance. @@ -301,14 +470,14 @@ pub struct AuthEntryReference { _phantom: std::marker::PhantomData, } -impl std::fmt::Debug for AuthEntryReference { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl fmt::Debug for AuthEntryReference { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "AuthEntryReference({})", self.key) } } -impl std::fmt::Display for AuthEntryReference { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for AuthEntryReference { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "AuthEntryReference({})", self.key) } } @@ -362,3 +531,16 @@ impl std::hash::Hash for AuthEntryReference { self.key.hash(state); } } + +// Helper function to format EnrichedValueType +fn format_value_type(value_type: &EnrichedValueType) -> String { + let mut typ = match &value_type.typ { + ValueType::Basic(basic) => format!("{}", basic), + ValueType::Table(t) => format!("{}", t.kind), + ValueType::Struct(s) => format!("{}", s), + }; + if value_type.nullable { + typ.push('?'); + } + typ +} diff --git a/src/py/mod.rs b/src/py/mod.rs index c02001ca..9d1858f9 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -2,6 +2,7 @@ use crate::prelude::*; use crate::base::schema::{FieldSchema, ValueType}; use crate::base::spec::VectorSimilarityMetric; +use crate::base::spec::{NamedSpec, ReactiveOpSpec}; use crate::execution::query; use crate::lib_context::{clear_lib_context, get_auth_registry, init_lib_context}; use crate::ops::interface::{QueryResult, QueryResults}; @@ -196,6 +197,58 @@ impl Flow { }) } + pub fn get_spec(&self) -> Vec<(String, String, u32)> { + let spec = &self.0.flow.flow_instance; + let mut result = Vec::new(); + + // Header + result.push(("Header".to_string(), format!("Flow: {}", spec.name), 0)); + + // Sources + for op in &spec.import_ops { + result.push(("Sources".to_string(), op.to_string(), 0)); + } + + // Processing + fn process_reactive_op( + op: &NamedSpec, + result: &mut Vec<(String, String, u32)>, + indent: u32, + ) { + result.push(("Processing".to_string(), op.to_string(), indent)); + if let ReactiveOpSpec::ForEach(fe) = &op.spec { + result.push(( + "Processing".to_string(), + fe.op_scope.to_string(), + indent + 1, + )); + for nested_op in &fe.op_scope.ops { + process_reactive_op(nested_op, result, indent + 2); + } + } + } + + for op in &spec.reactive_ops { + process_reactive_op(op, &mut result, 0); + } + + // Targets + for op in &spec.export_ops { + result.push(("Targets".to_string(), op.to_string(), 0)); + } + + // Declarations + for decl in &spec.declarations { + result.push(( + "Declarations".to_string(), + format!("Declaration: {}", decl), + 0, + )); + } + + result + } + pub fn get_schema(&self) -> Vec<(String, String, String)> { let schema = &self.0.flow.data_schema; let mut result = Vec::new(); @@ -211,7 +264,7 @@ impl Flow { let mut field_type = match &field.value_type.typ { ValueType::Basic(basic) => format!("{}", basic), ValueType::Table(t) => format!("{}", t.kind), - ValueType::Struct(_) => "Struct".to_string(), + ValueType::Struct(s) => format!("{}", s), }; if field.value_type.nullable { From 0c66e1c86986a333f22d6aae48c2cfd3a619a151 Mon Sep 17 00:00:00 2001 From: lemorage Date: Wed, 7 May 2025 13:21:42 +0200 Subject: [PATCH 2/8] feat(py): add verbose option for detailed output in spec getter --- src/base/spec.rs | 240 ++++++++++++++++++++++++++++++++++++----------- src/py/mod.rs | 106 +++++++++++++++------ 2 files changed, 262 insertions(+), 84 deletions(-) diff --git a/src/base/spec.rs b/src/base/spec.rs index eaa2da97..215cb7a5 100644 --- a/src/base/spec.rs +++ b/src/base/spec.rs @@ -1,7 +1,8 @@ use crate::prelude::*; -use super::schema::{EnrichedValueType, FieldSchema, ValueType}; +use super::schema::{EnrichedValueType, FieldSchema}; use serde::{Deserialize, Serialize}; +use serde_json::Value; use std::fmt; use std::ops::Deref; @@ -115,9 +116,8 @@ pub struct ConstantMapping { impl fmt::Display for ConstantMapping { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let schema = format_value_type(&self.schema); let value = serde_json::to_string(&self.value).unwrap_or("#serde_error".to_string()); - write!(f, "Constant({}: {})", value, schema) + write!(f, "{}", value) } } @@ -137,10 +137,10 @@ impl fmt::Display for StructMapping { let fields = self .fields .iter() - .map(|field| format!("{}={}", field.name, field.spec)) + .map(|field| field.name.clone()) .collect::>() - .join(", "); - write!(f, "[{}]", fields) + .join(","); + write!(f, "{}", fields) } } @@ -216,11 +216,41 @@ pub struct OpSpec { pub spec: serde_json::Map, } -impl fmt::Display for OpSpec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl OpSpec { + pub fn format_concise(&self) -> String { + let mut parts = vec![]; + for (key, value) in self.spec.iter() { + match value { + Value::String(s) => parts.push(format!("{}={}", key, s)), + Value::Array(arr) => { + let items = arr + .iter() + .filter_map(|v| v.as_str()) + .collect::>() + .join(","); + if !items.is_empty() { + parts.push(format!("{}={}", key, items)); + } + } + Value::Object(obj) => { + if let Some(model) = obj.get("model").and_then(|v| v.as_str()) { + parts.push(format!("{}={}", key, model)); + } + } + _ => {} + } + } + if parts.is_empty() { + self.kind.clone() + } else { + format!("{}({})", self.kind, parts.join(", ")) + } + } + + pub fn format_verbose(&self) -> String { let spec_str = serde_json::to_string_pretty(&self.spec) .map(|s| { - let lines: Vec<&str> = s.lines().take(50).collect(); + let lines: Vec<&str> = s.lines().collect(); if lines.len() < s.lines().count() { lines .into_iter() @@ -232,7 +262,13 @@ impl fmt::Display for OpSpec { } }) .unwrap_or("#serde_error".to_string()); - write!(f, "OpSpec: kind={}, spec={}", self.kind, spec_str) + format!("{}({})", self.kind, spec_str) + } +} + +impl fmt::Display for OpSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.format_concise()) } } @@ -246,7 +282,7 @@ impl fmt::Display for SourceRefreshOptions { let refresh = self .refresh_interval .map(|d| format!("{:?}", d)) - .unwrap_or("None".to_string()); + .unwrap_or("none".to_string()); write!(f, "{}", refresh) } } @@ -259,13 +295,28 @@ pub struct ImportOpSpec { pub refresh_options: SourceRefreshOptions, } +impl ImportOpSpec { + fn format(&self, verbose: bool) -> String { + let source = if verbose { + self.source.format_verbose() + } else { + self.source.format_concise() + }; + format!("source={}, refresh={}", source, self.refresh_options) + } + + pub fn format_concise(&self) -> String { + self.format(false) + } + + pub fn format_verbose(&self) -> String { + self.format(true) + } +} + impl fmt::Display for ImportOpSpec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "Import: source={}, refresh={}", - self.source, self.refresh_options - ) + write!(f, "{}", self.format_concise()) } } @@ -275,15 +326,40 @@ pub struct TransformOpSpec { pub op: OpSpec, } -impl fmt::Display for TransformOpSpec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl TransformOpSpec { + fn format(&self, verbose: bool) -> String { let inputs = self .inputs .iter() - .map(|input| input.to_string()) + .map(ToString::to_string) .collect::>() - .join(", "); - write!(f, "Transform: op={}, inputs=[{}]", self.op, inputs) + .join(","); + + let op_str = if verbose { + self.op.format_verbose() + } else { + self.op.format_concise() + }; + + if verbose { + format!("op={}, inputs=[{}]", op_str, inputs) + } else { + format!("op={}, inputs={}", op_str, inputs) + } + } + + pub fn format_concise(&self) -> String { + self.format(false) + } + + pub fn format_verbose(&self) -> String { + self.format(true) + } +} + +impl fmt::Display for TransformOpSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.format_concise()) } } @@ -294,9 +370,15 @@ pub struct ForEachOpSpec { pub op_scope: ReactiveOpScope, } +impl ForEachOpSpec { + pub fn get_label(&self) -> String { + format!("Loop over {}", self.field_path) + } +} + impl fmt::Display for ForEachOpSpec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "ForEach: field={}", self.field_path) + write!(f, "field={}", self.field_path) } } @@ -313,13 +395,35 @@ pub struct CollectOpSpec { pub auto_uuid_field: Option, } +impl CollectOpSpec { + fn format(&self, verbose: bool) -> String { + let uuid = self.auto_uuid_field.as_deref().unwrap_or("none"); + + if verbose { + format!( + "scope={}, collector={}, input=[{}], uuid={}", + self.scope_name, self.collector_name, self.input, uuid + ) + } else { + format!( + "collector={}, input={}, uuid={}", + self.collector_name, self.input, uuid + ) + } + } + + pub fn format_concise(&self) -> String { + self.format(false) + } + + pub fn format_verbose(&self) -> String { + self.format(true) + } +} + impl fmt::Display for CollectOpSpec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "Collect: scope={}, collector={}, input={}, uuid_field={:?}", - self.scope_name, self.collector_name, self.input, self.auto_uuid_field - ) + write!(f, "{}", self.format_concise()) } } @@ -365,19 +469,15 @@ impl fmt::Display for IndexOptions { let primary_keys = self .primary_key_fields .as_ref() - .map(|p| p.join(", ")) + .map(|p| p.join(",")) .unwrap_or_default(); let vector_indexes = self .vector_indexes .iter() .map(|v| v.to_string()) .collect::>() - .join(", "); - write!( - f, - "IndexOptions: primary_keys=[{}], vector_indexes=[{}]", - primary_keys, vector_indexes - ) + .join(","); + write!(f, "keys={}, indexes={}", primary_keys, vector_indexes) } } @@ -389,13 +489,38 @@ pub struct ExportOpSpec { pub setup_by_user: bool, } +impl ExportOpSpec { + fn format(&self, verbose: bool) -> String { + let target_str = if verbose { + self.target.format_verbose() + } else { + self.target.format_concise() + }; + + let base = format!( + "collector={}, target={}, {}", + self.collector_name, target_str, self.index_options + ); + + if verbose { + format!("{}, setup_by_user={}", base, self.setup_by_user) + } else { + base + } + } + + pub fn format_concise(&self) -> String { + self.format(false) + } + + pub fn format_verbose(&self) -> String { + self.format(true) + } +} + impl fmt::Display for ExportOpSpec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "Export: collector={}, target={}, {}, setup_by_user={}", - self.collector_name, self.target, self.index_options, self.setup_by_user - ) + write!(f, "{}", self.format_concise()) } } @@ -407,16 +532,30 @@ pub enum ReactiveOpSpec { Collect(CollectOpSpec), } -impl fmt::Display for ReactiveOpSpec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl ReactiveOpSpec { + pub fn format_concise(&self) -> String { + match self { + ReactiveOpSpec::Transform(t) => format!("Transform: {}", t.format_concise()), + ReactiveOpSpec::ForEach(fe) => format!("{}", fe.get_label()), + ReactiveOpSpec::Collect(c) => c.format_concise(), + } + } + + pub fn format_verbose(&self) -> String { match self { - ReactiveOpSpec::Transform(t) => write!(f, "{}", t), - ReactiveOpSpec::ForEach(fe) => write!(f, "{}", fe), - ReactiveOpSpec::Collect(c) => write!(f, "{}", c), + ReactiveOpSpec::Transform(t) => format!("Transform: {}", t.format_verbose()), + ReactiveOpSpec::ForEach(fe) => format!("ForEach: {}", fe), + ReactiveOpSpec::Collect(c) => format!("Collect: {}", c.format_verbose()), } } } +impl fmt::Display for ReactiveOpSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.format_concise()) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReactiveOpScope { pub name: ScopeName, @@ -531,16 +670,3 @@ impl std::hash::Hash for AuthEntryReference { self.key.hash(state); } } - -// Helper function to format EnrichedValueType -fn format_value_type(value_type: &EnrichedValueType) -> String { - let mut typ = match &value_type.typ { - ValueType::Basic(basic) => format!("{}", basic), - ValueType::Table(t) => format!("{}", t.kind), - ValueType::Struct(s) => format!("{}", s), - }; - if value_type.nullable { - typ.push('?'); - } - typ -} diff --git a/src/py/mod.rs b/src/py/mod.rs index 9d1858f9..4a540731 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -197,54 +197,106 @@ impl Flow { }) } - pub fn get_spec(&self) -> Vec<(String, String, u32)> { + #[pyo3(signature = (verbose=false))] + pub fn get_spec(&self, verbose: bool) -> Vec<(String, String, u32)> { let spec = &self.0.flow.flow_instance; - let mut result = Vec::new(); + let mut result = Vec::with_capacity( + 1 + spec.import_ops.len() + + spec.reactive_ops.len() + + spec.export_ops.len() + + spec.declarations.len(), + ); + + fn extend_with( + out: &mut Vec<(String, String, u32)>, + label: &'static str, + items: I, + f: F, + ) where + I: IntoIterator, + F: Fn(&T) -> String, + { + out.extend(items.into_iter().map(|item| (label.into(), f(&item), 0))); + } // Header - result.push(("Header".to_string(), format!("Flow: {}", spec.name), 0)); + result.push(("Header".into(), format!("Flow: {}", spec.name), 0)); // Sources - for op in &spec.import_ops { - result.push(("Sources".to_string(), op.to_string(), 0)); - } + extend_with(&mut result, "Sources", spec.import_ops.iter(), |op| { + format!( + "Import: name={}, {}", + op.name, + if verbose { + op.spec.format_verbose() + } else { + op.spec.format_concise() + } + ) + }); // Processing - fn process_reactive_op( + fn walk( op: &NamedSpec, - result: &mut Vec<(String, String, u32)>, indent: u32, + verbose: bool, + out: &mut Vec<(String, String, u32)>, ) { - result.push(("Processing".to_string(), op.to_string(), indent)); + out.push(( + "Processing".into(), + format!( + "{}: {}", + op.name, + if verbose { + op.spec.format_verbose() + } else { + op.spec.format_concise() + } + ), + indent, + )); + if let ReactiveOpSpec::ForEach(fe) = &op.spec { - result.push(( - "Processing".to_string(), - fe.op_scope.to_string(), - indent + 1, - )); - for nested_op in &fe.op_scope.ops { - process_reactive_op(nested_op, result, indent + 2); + out.push(("Processing".into(), fe.op_scope.to_string(), indent + 1)); + for nested in &fe.op_scope.ops { + walk(nested, indent + 2, verbose, out); } } } for op in &spec.reactive_ops { - process_reactive_op(op, &mut result, 0); + walk(op, 0, verbose, &mut result); } // Targets - for op in &spec.export_ops { - result.push(("Targets".to_string(), op.to_string(), 0)); - } + extend_with(&mut result, "Targets", spec.export_ops.iter(), |op| { + format!( + "Export: name={}, {}", + op.name, + if verbose { + op.spec.format_verbose() + } else { + op.spec.format_concise() + } + ) + }); // Declarations - for decl in &spec.declarations { - result.push(( - "Declarations".to_string(), - format!("Declaration: {}", decl), - 0, - )); - } + extend_with( + &mut result, + "Declarations", + spec.declarations.iter(), + |decl| { + format!( + "Declaration: {}", + if verbose { + decl.format_verbose() + } else { + decl.format_concise() + } + ) + }, + ); result } From ab49815e0bf81c8ac7c130dbaa74eee5283bd30e Mon Sep 17 00:00:00 2001 From: lemorage Date: Wed, 7 May 2025 13:46:36 +0200 Subject: [PATCH 3/8] feat(cli): display flow spec with verbose option and tree structured output --- python/cocoindex/cli.py | 13 +++--- python/cocoindex/flow.py | 89 ++++++++++++++++++---------------------- 2 files changed, 47 insertions(+), 55 deletions(-) diff --git a/python/cocoindex/cli.py b/python/cocoindex/cli.py index 5c95be7f..a119bc8c 100644 --- a/python/cocoindex/cli.py +++ b/python/cocoindex/cli.py @@ -55,16 +55,17 @@ def ls(show_all: bool): @cli.command() @click.argument("flow_name", type=str, required=False) -@click.option("--color/--no-color", default=True) -def show(flow_name: str | None, color: bool): +@click.option("--color/--no-color", default=True, help="Enable or disable colored output.") +@click.option("--verbose", is_flag=True, help="Show verbose output with full details.") +def show(flow_name: str | None, color: bool, verbose: bool): """ - Show the flow spec in a readable format with colored output, - including the schema. + Show the flow spec and schema in a readable format with colored output. """ flow = _flow_by_name(flow_name) console = Console(no_color=not color) - console.print(flow._render_text()) + console.print(flow._render_spec(verbose=verbose)) + console.print() table = Table( title=f"Schema for Flow: {flow.name}", show_header=True, @@ -74,7 +75,7 @@ def show(flow_name: str | None, color: bool): table.add_column("Type", style="green") table.add_column("Attributes", style="yellow") - for field_name, field_type, attr_str in flow._render_schema(): + for field_name, field_type, attr_str in flow._get_schema(): table.add_row(field_name, field_type, attr_str) console.print(table) diff --git a/python/cocoindex/flow.py b/python/cocoindex/flow.py index cbc095ea..d8de05eb 100644 --- a/python/cocoindex/flow.py +++ b/python/cocoindex/flow.py @@ -15,7 +15,7 @@ from enum import Enum from dataclasses import dataclass from rich.text import Text -from rich.console import Console +from rich.tree import Tree from . import _engine from . import index @@ -454,61 +454,52 @@ def _lazy_engine_flow() -> _engine.Flow: return engine_flow self._lazy_engine_flow = _lazy_engine_flow - def _format_flow(self, flow_dict: dict) -> Text: - output = Text() + def _render_spec(self, verbose: bool = False) -> Tree: + """ + Render the flow spec as a styled rich Tree with hierarchical structure. + """ + tree = Tree(f"Flow: {self.name}", style="cyan") + current_section = None + section_node = None + indent_stack = [] - def add_line(content, indent=0, style=None, end="\n"): - output.append(" " * indent) - output.append(content, style=style) - output.append(end) + for i, (section, content, indent) in enumerate(self._get_spec(verbose=verbose)): + # Skip "Scope" entries (see ReactiveOpScope in spec.rs) + if content.startswith("Scope:"): + continue - def format_key_value(key, value, indent): - if isinstance(value, (dict, list)): - add_line(f"- {key}:", indent, style="green") - format_data(value, indent + 2) - else: - add_line(f"- {key}:", indent, style="green", end="") - add_line(f" {value}", style="yellow") - - def format_data(data, indent=0): - if isinstance(data, dict): - for key, value in data.items(): - format_key_value(key, value, indent) - elif isinstance(data, list): - for i, item in enumerate(data): - format_key_value(f"[{i}]", item, indent) + if section != current_section: + current_section = section + section_node = tree.add(f"{section}:", style="bold magenta") + indent_stack = [(0, section_node)] + + while indent_stack and indent_stack[-1][0] >= indent: + indent_stack.pop() + + parent = indent_stack[-1][1] if indent_stack else section_node + styled_content = Text(content, style="yellow") + is_parent = any( + next_indent > indent + for _, next_content, next_indent in self._get_spec(verbose=verbose)[i + 1:] + if not next_content.startswith("Scope:") + ) + + if is_parent: + node = parent.add(styled_content, style=None) + indent_stack.append((indent, node)) else: - add_line(str(data), indent, style="yellow") - - # Header - flow_name = flow_dict.get("name", "Unnamed") - add_line(f"Flow: {flow_name}", style="bold cyan") - - # Section - for section_title, section_key in [ - ("Sources:", "import_ops"), - ("Processing:", "reactive_ops"), - ("Targets:", "export_ops"), - ]: - add_line("") - add_line(section_title, style="bold cyan") - format_data(flow_dict.get(section_key, []), indent=0) - - return output - - def _render_text(self) -> Text: - flow_spec_str = str(self._lazy_engine_flow()) - try: - flow_dict = json.loads(flow_spec_str) - return self._format_flow(flow_dict) - except json.JSONDecodeError: - return Text(flow_spec_str) + parent.add(styled_content, style=None) + + return tree + + def _get_spec(self, verbose: bool = False) -> list[tuple[str, str, int]]: + return self._lazy_engine_flow().get_spec(verbose=verbose) - def _render_schema(self) -> list[tuple[str, str, str]]: + def _get_schema(self) -> list[tuple[str, str, str]]: return self._lazy_engine_flow().get_schema() def __str__(self): - return str(self._render_text()) + return str(self._get_spec()) def __repr__(self): return repr(self._lazy_engine_flow()) From 945cb519abf832df68146dabb56f587fa06c9e3e Mon Sep 17 00:00:00 2001 From: lemorage Date: Thu, 8 May 2025 09:21:25 +0200 Subject: [PATCH 4/8] refactor: update spec retrieval to use enum for concise/verbose outputs --- python/cocoindex/flow.py | 2 +- src/base/spec.rs | 163 ++++++++++++++++++--------------------- src/py/mod.rs | 58 ++++---------- 3 files changed, 90 insertions(+), 133 deletions(-) diff --git a/python/cocoindex/flow.py b/python/cocoindex/flow.py index d8de05eb..dd2bea10 100644 --- a/python/cocoindex/flow.py +++ b/python/cocoindex/flow.py @@ -493,7 +493,7 @@ def _render_spec(self, verbose: bool = False) -> Tree: return tree def _get_spec(self, verbose: bool = False) -> list[tuple[str, str, int]]: - return self._lazy_engine_flow().get_spec(verbose=verbose) + return self._lazy_engine_flow().get_spec(format_mode="verbose" if verbose else "concise") def _get_schema(self) -> list[tuple[str, str, str]]: return self._lazy_engine_flow().get_schema() diff --git a/src/base/spec.rs b/src/base/spec.rs index 215cb7a5..0264d6cb 100644 --- a/src/base/spec.rs +++ b/src/base/spec.rs @@ -6,6 +6,23 @@ use serde_json::Value; use std::fmt; use std::ops::Deref; +// Define SpecFormatMode enum for type-safe formatting +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SpecFormatMode { + Concise, + Verbose, +} + +impl SpecFormatMode { + pub fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "concise" => Ok(SpecFormatMode::Concise), + "verbose" => Ok(SpecFormatMode::Verbose), + _ => Err(format!("Invalid format mode: {}", s)), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "kind")] pub enum SpecString { @@ -264,6 +281,13 @@ impl OpSpec { .unwrap_or("#serde_error".to_string()); format!("{}({})", self.kind, spec_str) } + + pub fn format(&self, mode: SpecFormatMode) -> String { + match mode { + SpecFormatMode::Concise => self.format_concise(), + SpecFormatMode::Verbose => self.format_verbose(), + } + } } impl fmt::Display for OpSpec { @@ -296,27 +320,18 @@ pub struct ImportOpSpec { } impl ImportOpSpec { - fn format(&self, verbose: bool) -> String { - let source = if verbose { - self.source.format_verbose() - } else { - self.source.format_concise() + pub fn format(&self, mode: SpecFormatMode) -> String { + let source = match mode { + SpecFormatMode::Concise => self.source.format_concise(), + SpecFormatMode::Verbose => self.source.format_verbose(), }; format!("source={}, refresh={}", source, self.refresh_options) } - - pub fn format_concise(&self) -> String { - self.format(false) - } - - pub fn format_verbose(&self) -> String { - self.format(true) - } } impl fmt::Display for ImportOpSpec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.format_concise()) + write!(f, "{}", self.format(SpecFormatMode::Concise)) } } @@ -327,39 +342,27 @@ pub struct TransformOpSpec { } impl TransformOpSpec { - fn format(&self, verbose: bool) -> String { + pub fn format(&self, mode: SpecFormatMode) -> String { let inputs = self .inputs .iter() .map(ToString::to_string) .collect::>() .join(","); - - let op_str = if verbose { - self.op.format_verbose() - } else { - self.op.format_concise() + let op_str = match mode { + SpecFormatMode::Concise => self.op.format_concise(), + SpecFormatMode::Verbose => self.op.format_verbose(), }; - - if verbose { - format!("op={}, inputs=[{}]", op_str, inputs) - } else { - format!("op={}, inputs={}", op_str, inputs) + match mode { + SpecFormatMode::Concise => format!("op={}, inputs={}", op_str, inputs), + SpecFormatMode::Verbose => format!("op={}, inputs=[{}]", op_str, inputs), } } - - pub fn format_concise(&self) -> String { - self.format(false) - } - - pub fn format_verbose(&self) -> String { - self.format(true) - } } impl fmt::Display for TransformOpSpec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.format_concise()) + write!(f, "{}", self.format(SpecFormatMode::Concise)) } } @@ -374,6 +377,13 @@ impl ForEachOpSpec { pub fn get_label(&self) -> String { format!("Loop over {}", self.field_path) } + + pub fn format(&self, mode: SpecFormatMode) -> String { + match mode { + SpecFormatMode::Concise => self.get_label(), + SpecFormatMode::Verbose => format!("field={}", self.field_path), + } + } } impl fmt::Display for ForEachOpSpec { @@ -396,34 +406,28 @@ pub struct CollectOpSpec { } impl CollectOpSpec { - fn format(&self, verbose: bool) -> String { + pub fn format(&self, mode: SpecFormatMode) -> String { let uuid = self.auto_uuid_field.as_deref().unwrap_or("none"); - - if verbose { - format!( - "scope={}, collector={}, input=[{}], uuid={}", - self.scope_name, self.collector_name, self.input, uuid - ) - } else { - format!( - "collector={}, input={}, uuid={}", - self.collector_name, self.input, uuid - ) + match mode { + SpecFormatMode::Concise => { + format!( + "collector={}, input={}, uuid={}", + self.collector_name, self.input, uuid + ) + } + SpecFormatMode::Verbose => { + format!( + "scope={}, collector={}, input=[{}], uuid={}", + self.scope_name, self.collector_name, self.input, uuid + ) + } } } - - pub fn format_concise(&self) -> String { - self.format(false) - } - - pub fn format_verbose(&self) -> String { - self.format(true) - } } impl fmt::Display for CollectOpSpec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.format_concise()) + write!(f, "{}", self.format(SpecFormatMode::Concise)) } } @@ -490,37 +494,25 @@ pub struct ExportOpSpec { } impl ExportOpSpec { - fn format(&self, verbose: bool) -> String { - let target_str = if verbose { - self.target.format_verbose() - } else { - self.target.format_concise() + pub fn format(&self, mode: SpecFormatMode) -> String { + let target_str = match mode { + SpecFormatMode::Concise => self.target.format_concise(), + SpecFormatMode::Verbose => self.target.format_verbose(), }; - let base = format!( "collector={}, target={}, {}", self.collector_name, target_str, self.index_options ); - - if verbose { - format!("{}, setup_by_user={}", base, self.setup_by_user) - } else { - base + match mode { + SpecFormatMode::Concise => base, + SpecFormatMode::Verbose => format!("{}, setup_by_user={}", base, self.setup_by_user), } } - - pub fn format_concise(&self) -> String { - self.format(false) - } - - pub fn format_verbose(&self) -> String { - self.format(true) - } } impl fmt::Display for ExportOpSpec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.format_concise()) + write!(f, "{}", self.format(SpecFormatMode::Concise)) } } @@ -533,26 +525,21 @@ pub enum ReactiveOpSpec { } impl ReactiveOpSpec { - pub fn format_concise(&self) -> String { + pub fn format(&self, mode: SpecFormatMode) -> String { match self { - ReactiveOpSpec::Transform(t) => format!("Transform: {}", t.format_concise()), - ReactiveOpSpec::ForEach(fe) => format!("{}", fe.get_label()), - ReactiveOpSpec::Collect(c) => c.format_concise(), - } - } - - pub fn format_verbose(&self) -> String { - match self { - ReactiveOpSpec::Transform(t) => format!("Transform: {}", t.format_verbose()), - ReactiveOpSpec::ForEach(fe) => format!("ForEach: {}", fe), - ReactiveOpSpec::Collect(c) => format!("Collect: {}", c.format_verbose()), + ReactiveOpSpec::Transform(t) => format!("Transform: {}", t.format(mode)), + ReactiveOpSpec::ForEach(fe) => match mode { + SpecFormatMode::Concise => format!("{}", fe.get_label()), + SpecFormatMode::Verbose => format!("ForEach: {}", fe.format(mode)), + }, + ReactiveOpSpec::Collect(c) => format!("Collect: {}", c.format(mode)), } } } impl fmt::Display for ReactiveOpSpec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.format_concise()) + write!(f, "{}", self.format(SpecFormatMode::Concise)) } } diff --git a/src/py/mod.rs b/src/py/mod.rs index 4a540731..09935c37 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -1,6 +1,7 @@ use crate::prelude::*; use crate::base::schema::{FieldSchema, ValueType}; +use crate::base::spec::SpecFormatMode; use crate::base::spec::VectorSimilarityMetric; use crate::base::spec::{NamedSpec, ReactiveOpSpec}; use crate::execution::query; @@ -197,8 +198,10 @@ impl Flow { }) } - #[pyo3(signature = (verbose=false))] - pub fn get_spec(&self, verbose: bool) -> Vec<(String, String, u32)> { + #[pyo3(signature = (format_mode="concise"))] + pub fn get_spec(&self, format_mode: &str) -> PyResult> { + let mode = SpecFormatMode::from_str(format_mode) + .map_err(|e| PyErr::new::(e))?; let spec = &self.0.flow.flow_instance; let mut result = Vec::with_capacity( 1 + spec.import_ops.len() @@ -224,61 +227,37 @@ impl Flow { // Sources extend_with(&mut result, "Sources", spec.import_ops.iter(), |op| { - format!( - "Import: name={}, {}", - op.name, - if verbose { - op.spec.format_verbose() - } else { - op.spec.format_concise() - } - ) + format!("Import: name={}, {}", op.name, op.spec.format(mode)) }); // Processing fn walk( op: &NamedSpec, indent: u32, - verbose: bool, + mode: SpecFormatMode, out: &mut Vec<(String, String, u32)>, ) { out.push(( "Processing".into(), - format!( - "{}: {}", - op.name, - if verbose { - op.spec.format_verbose() - } else { - op.spec.format_concise() - } - ), + format!("{}: {}", op.name, op.spec.format(mode)), indent, )); if let ReactiveOpSpec::ForEach(fe) = &op.spec { out.push(("Processing".into(), fe.op_scope.to_string(), indent + 1)); for nested in &fe.op_scope.ops { - walk(nested, indent + 2, verbose, out); + walk(nested, indent + 2, mode, out); } } } for op in &spec.reactive_ops { - walk(op, 0, verbose, &mut result); + walk(op, 0, mode, &mut result); } // Targets extend_with(&mut result, "Targets", spec.export_ops.iter(), |op| { - format!( - "Export: name={}, {}", - op.name, - if verbose { - op.spec.format_verbose() - } else { - op.spec.format_concise() - } - ) + format!("Export: name={}, {}", op.name, op.spec.format(mode)) }); // Declarations @@ -286,19 +265,10 @@ impl Flow { &mut result, "Declarations", spec.declarations.iter(), - |decl| { - format!( - "Declaration: {}", - if verbose { - decl.format_verbose() - } else { - decl.format_concise() - } - ) - }, + |decl| format!("Declaration: {}", decl.format(mode)), ); - result + Ok(result) } pub fn get_schema(&self) -> Vec<(String, String, String)> { @@ -316,7 +286,7 @@ impl Flow { let mut field_type = match &field.value_type.typ { ValueType::Basic(basic) => format!("{}", basic), ValueType::Table(t) => format!("{}", t.kind), - ValueType::Struct(s) => format!("{}", s), + ValueType::Struct(_) => "Struct".to_string(), }; if field.value_type.nullable { From f77961deb693f7dc621eda65a38b28a177ebfb5d Mon Sep 17 00:00:00 2001 From: lemorage Date: Thu, 8 May 2025 16:29:16 +0200 Subject: [PATCH 5/8] refactor: update spec formatting logic to use `SpecFormatter` trait --- python/cocoindex/flow.py | 2 +- src/base/spec.rs | 198 ++++++++++++++------------------------- src/py/mod.rs | 11 +-- 3 files changed, 74 insertions(+), 137 deletions(-) diff --git a/python/cocoindex/flow.py b/python/cocoindex/flow.py index dd2bea10..e8cdbb10 100644 --- a/python/cocoindex/flow.py +++ b/python/cocoindex/flow.py @@ -493,7 +493,7 @@ def _render_spec(self, verbose: bool = False) -> Tree: return tree def _get_spec(self, verbose: bool = False) -> list[tuple[str, str, int]]: - return self._lazy_engine_flow().get_spec(format_mode="verbose" if verbose else "concise") + return self._lazy_engine_flow().get_spec(output_mode="verbose" if verbose else "concise") def _get_schema(self) -> list[tuple[str, str, str]]: return self._lazy_engine_flow().get_schema() diff --git a/src/base/spec.rs b/src/base/spec.rs index 0264d6cb..e39e0432 100644 --- a/src/base/spec.rs +++ b/src/base/spec.rs @@ -2,27 +2,34 @@ use crate::prelude::*; use super::schema::{EnrichedValueType, FieldSchema}; use serde::{Deserialize, Serialize}; -use serde_json::Value; use std::fmt; use std::ops::Deref; -// Define SpecFormatMode enum for type-safe formatting +/// OutputMode enum for displaying spec info in different granularity #[derive(Debug, Clone, Copy, PartialEq)] -pub enum SpecFormatMode { +pub enum OutputMode { Concise, Verbose, } -impl SpecFormatMode { - pub fn from_str(s: &str) -> Result { +impl OutputMode { + pub fn from_str(s: &str) -> Self { match s.to_lowercase().as_str() { - "concise" => Ok(SpecFormatMode::Concise), - "verbose" => Ok(SpecFormatMode::Verbose), - _ => Err(format!("Invalid format mode: {}", s)), + "concise" => OutputMode::Concise, + "verbose" => OutputMode::Verbose, + _ => unreachable!( + "Invalid format mode: {}. Expected 'concise' or 'verbose'.", + s + ), } } } +/// Formatting spec per output mode +pub trait SpecFormatter { + fn format(&self, mode: OutputMode) -> String; +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "kind")] pub enum SpecString { @@ -233,69 +240,31 @@ pub struct OpSpec { pub spec: serde_json::Map, } -impl OpSpec { - pub fn format_concise(&self) -> String { - let mut parts = vec![]; - for (key, value) in self.spec.iter() { - match value { - Value::String(s) => parts.push(format!("{}={}", key, s)), - Value::Array(arr) => { - let items = arr - .iter() - .filter_map(|v| v.as_str()) - .collect::>() - .join(","); - if !items.is_empty() { - parts.push(format!("{}={}", key, items)); - } - } - Value::Object(obj) => { - if let Some(model) = obj.get("model").and_then(|v| v.as_str()) { - parts.push(format!("{}={}", key, model)); - } - } - _ => {} - } - } - if parts.is_empty() { - self.kind.clone() - } else { - format!("{}({})", self.kind, parts.join(", ")) - } - } - - pub fn format_verbose(&self) -> String { - let spec_str = serde_json::to_string_pretty(&self.spec) - .map(|s| { - let lines: Vec<&str> = s.lines().collect(); - if lines.len() < s.lines().count() { - lines - .into_iter() - .chain(["..."]) - .collect::>() - .join("\n ") - } else { - lines.join("\n ") - } - }) - .unwrap_or("#serde_error".to_string()); - format!("{}({})", self.kind, spec_str) - } - - pub fn format(&self, mode: SpecFormatMode) -> String { +impl SpecFormatter for OpSpec { + fn format(&self, mode: OutputMode) -> String { match mode { - SpecFormatMode::Concise => self.format_concise(), - SpecFormatMode::Verbose => self.format_verbose(), + OutputMode::Concise => self.kind.clone(), + OutputMode::Verbose => { + let spec_str = serde_json::to_string_pretty(&self.spec) + .map(|s| { + let lines: Vec<&str> = s.lines().collect(); + if lines.len() < s.lines().count() { + lines + .into_iter() + .chain(["..."]) + .collect::>() + .join("\n ") + } else { + lines.join("\n ") + } + }) + .unwrap_or("#serde_error".to_string()); + format!("{}({})", self.kind, spec_str) + } } } } -impl fmt::Display for OpSpec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.format_concise()) - } -} - #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct SourceRefreshOptions { pub refresh_interval: Option, @@ -319,53 +288,43 @@ pub struct ImportOpSpec { pub refresh_options: SourceRefreshOptions, } -impl ImportOpSpec { - pub fn format(&self, mode: SpecFormatMode) -> String { - let source = match mode { - SpecFormatMode::Concise => self.source.format_concise(), - SpecFormatMode::Verbose => self.source.format_verbose(), - }; +impl SpecFormatter for ImportOpSpec { + fn format(&self, mode: OutputMode) -> String { + let source = self.source.format(mode); format!("source={}, refresh={}", source, self.refresh_options) } } impl fmt::Display for ImportOpSpec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.format(SpecFormatMode::Concise)) + write!(f, "{}", self.format(OutputMode::Concise)) } } +/// Transform data using a given operator. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TransformOpSpec { pub inputs: Vec, pub op: OpSpec, } -impl TransformOpSpec { - pub fn format(&self, mode: SpecFormatMode) -> String { +impl SpecFormatter for TransformOpSpec { + fn format(&self, mode: OutputMode) -> String { let inputs = self .inputs .iter() .map(ToString::to_string) .collect::>() .join(","); - let op_str = match mode { - SpecFormatMode::Concise => self.op.format_concise(), - SpecFormatMode::Verbose => self.op.format_verbose(), - }; + let op_str = self.op.format(mode); match mode { - SpecFormatMode::Concise => format!("op={}, inputs={}", op_str, inputs), - SpecFormatMode::Verbose => format!("op={}, inputs=[{}]", op_str, inputs), + OutputMode::Concise => format!("op={}, inputs={}", op_str, inputs), + OutputMode::Verbose => format!("op={}, inputs=[{}]", op_str, inputs), } } } -impl fmt::Display for TransformOpSpec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.format(SpecFormatMode::Concise)) - } -} - +/// Apply reactive operations to each row of the input field. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ForEachOpSpec { /// Mapping that provides a table to apply reactive operations to. @@ -377,21 +336,18 @@ impl ForEachOpSpec { pub fn get_label(&self) -> String { format!("Loop over {}", self.field_path) } +} - pub fn format(&self, mode: SpecFormatMode) -> String { +impl SpecFormatter for ForEachOpSpec { + fn format(&self, mode: OutputMode) -> String { match mode { - SpecFormatMode::Concise => self.get_label(), - SpecFormatMode::Verbose => format!("field={}", self.field_path), + OutputMode::Concise => self.get_label(), + OutputMode::Verbose => format!("field={}", self.field_path), } } } -impl fmt::Display for ForEachOpSpec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "field={}", self.field_path) - } -} - +/// Emit data to a given collector at the given scope. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CollectOpSpec { /// Field values to be collected. @@ -405,17 +361,17 @@ pub struct CollectOpSpec { pub auto_uuid_field: Option, } -impl CollectOpSpec { - pub fn format(&self, mode: SpecFormatMode) -> String { +impl SpecFormatter for CollectOpSpec { + fn format(&self, mode: OutputMode) -> String { let uuid = self.auto_uuid_field.as_deref().unwrap_or("none"); match mode { - SpecFormatMode::Concise => { + OutputMode::Concise => { format!( "collector={}, input={}, uuid={}", self.collector_name, self.input, uuid ) } - SpecFormatMode::Verbose => { + OutputMode::Verbose => { format!( "scope={}, collector={}, input=[{}], uuid={}", self.scope_name, self.collector_name, self.input, uuid @@ -425,12 +381,6 @@ impl CollectOpSpec { } } -impl fmt::Display for CollectOpSpec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.format(SpecFormatMode::Concise)) - } -} - #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] pub enum VectorSimilarityMetric { CosineSimilarity, @@ -485,6 +435,7 @@ impl fmt::Display for IndexOptions { } } +/// Store data to a given sink. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ExportOpSpec { pub collector_name: FieldName, @@ -493,29 +444,21 @@ pub struct ExportOpSpec { pub setup_by_user: bool, } -impl ExportOpSpec { - pub fn format(&self, mode: SpecFormatMode) -> String { - let target_str = match mode { - SpecFormatMode::Concise => self.target.format_concise(), - SpecFormatMode::Verbose => self.target.format_verbose(), - }; +impl SpecFormatter for ExportOpSpec { + fn format(&self, mode: OutputMode) -> String { + let target_str = self.target.format(mode); let base = format!( "collector={}, target={}, {}", self.collector_name, target_str, self.index_options ); match mode { - SpecFormatMode::Concise => base, - SpecFormatMode::Verbose => format!("{}, setup_by_user={}", base, self.setup_by_user), + OutputMode::Concise => base, + OutputMode::Verbose => format!("{}, setup_by_user={}", base, self.setup_by_user), } } } -impl fmt::Display for ExportOpSpec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.format(SpecFormatMode::Concise)) - } -} - +/// A reactive operation reacts on given input values. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "action")] pub enum ReactiveOpSpec { @@ -524,25 +467,19 @@ pub enum ReactiveOpSpec { Collect(CollectOpSpec), } -impl ReactiveOpSpec { - pub fn format(&self, mode: SpecFormatMode) -> String { +impl SpecFormatter for ReactiveOpSpec { + fn format(&self, mode: OutputMode) -> String { match self { ReactiveOpSpec::Transform(t) => format!("Transform: {}", t.format(mode)), ReactiveOpSpec::ForEach(fe) => match mode { - SpecFormatMode::Concise => format!("{}", fe.get_label()), - SpecFormatMode::Verbose => format!("ForEach: {}", fe.format(mode)), + OutputMode::Concise => format!("{}", fe.get_label()), + OutputMode::Verbose => format!("ForEach: {}", fe.format(mode)), }, ReactiveOpSpec::Collect(c) => format!("Collect: {}", c.format(mode)), } } } -impl fmt::Display for ReactiveOpSpec { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.format(SpecFormatMode::Concise)) - } -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReactiveOpScope { pub name: ScopeName, @@ -556,6 +493,7 @@ impl fmt::Display for ReactiveOpScope { } } +/// A flow defines the rule to sync data from given sources to given sinks with given transformations. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FlowInstanceSpec { /// Name of the flow instance. diff --git a/src/py/mod.rs b/src/py/mod.rs index 09935c37..ca33503e 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -1,9 +1,9 @@ use crate::prelude::*; use crate::base::schema::{FieldSchema, ValueType}; -use crate::base::spec::SpecFormatMode; use crate::base::spec::VectorSimilarityMetric; use crate::base::spec::{NamedSpec, ReactiveOpSpec}; +use crate::base::spec::{OutputMode, SpecFormatter}; use crate::execution::query; use crate::lib_context::{clear_lib_context, get_auth_registry, init_lib_context}; use crate::ops::interface::{QueryResult, QueryResults}; @@ -198,10 +198,9 @@ impl Flow { }) } - #[pyo3(signature = (format_mode="concise"))] - pub fn get_spec(&self, format_mode: &str) -> PyResult> { - let mode = SpecFormatMode::from_str(format_mode) - .map_err(|e| PyErr::new::(e))?; + #[pyo3(signature = (output_mode="concise"))] + pub fn get_spec(&self, output_mode: &str) -> PyResult> { + let mode = OutputMode::from_str(output_mode); let spec = &self.0.flow.flow_instance; let mut result = Vec::with_capacity( 1 + spec.import_ops.len() @@ -234,7 +233,7 @@ impl Flow { fn walk( op: &NamedSpec, indent: u32, - mode: SpecFormatMode, + mode: OutputMode, out: &mut Vec<(String, String, u32)>, ) { out.push(( From c29e8fb2882831ff02df35db778559090e323310 Mon Sep 17 00:00:00 2001 From: lemorage Date: Fri, 9 May 2025 16:46:55 +0200 Subject: [PATCH 6/8] refactor: render spec line recursively on rust side --- python/cocoindex/flow.py | 39 ++++---------- src/base/spec.rs | 40 ++++++++------ src/py/mod.rs | 114 +++++++++++++++++++++++---------------- 3 files changed, 102 insertions(+), 91 deletions(-) diff --git a/python/cocoindex/flow.py b/python/cocoindex/flow.py index e8cdbb10..20c6d285 100644 --- a/python/cocoindex/flow.py +++ b/python/cocoindex/flow.py @@ -458,38 +458,19 @@ def _render_spec(self, verbose: bool = False) -> Tree: """ Render the flow spec as a styled rich Tree with hierarchical structure. """ + spec = self._get_spec(verbose=verbose) tree = Tree(f"Flow: {self.name}", style="cyan") - current_section = None - section_node = None - indent_stack = [] - - for i, (section, content, indent) in enumerate(self._get_spec(verbose=verbose)): - # Skip "Scope" entries (see ReactiveOpScope in spec.rs) - if content.startswith("Scope:"): - continue - - if section != current_section: - current_section = section - section_node = tree.add(f"{section}:", style="bold magenta") - indent_stack = [(0, section_node)] - - while indent_stack and indent_stack[-1][0] >= indent: - indent_stack.pop() - - parent = indent_stack[-1][1] if indent_stack else section_node - styled_content = Text(content, style="yellow") - is_parent = any( - next_indent > indent - for _, next_content, next_indent in self._get_spec(verbose=verbose)[i + 1:] - if not next_content.startswith("Scope:") - ) - if is_parent: - node = parent.add(styled_content, style=None) - indent_stack.append((indent, node)) - else: - parent.add(styled_content, style=None) + def build_tree(label: str, lines: list): + node = Tree(label, style="bold magenta" if lines else "cyan") + for line in lines: + child_node = node.add(Text(line.content, style="yellow")) + child_node.children = build_tree("", line.children).children + return node + for section, lines in spec.sections: + section_node = build_tree(f"{section}:", lines) + tree.children.append(section_node) return tree def _get_spec(self, verbose: bool = False) -> list[tuple[str, str, int]]: diff --git a/src/base/spec.rs b/src/base/spec.rs index e39e0432..7c75f960 100644 --- a/src/base/spec.rs +++ b/src/base/spec.rs @@ -1,35 +1,45 @@ use crate::prelude::*; use super::schema::{EnrichedValueType, FieldSchema}; +use pyo3::prelude::*; use serde::{Deserialize, Serialize}; use std::fmt; use std::ops::Deref; /// OutputMode enum for displaying spec info in different granularity -#[derive(Debug, Clone, Copy, PartialEq)] +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] pub enum OutputMode { Concise, Verbose, } -impl OutputMode { - pub fn from_str(s: &str) -> Self { - match s.to_lowercase().as_str() { - "concise" => OutputMode::Concise, - "verbose" => OutputMode::Verbose, - _ => unreachable!( - "Invalid format mode: {}. Expected 'concise' or 'verbose'.", - s - ), - } - } -} - /// Formatting spec per output mode pub trait SpecFormatter { fn format(&self, mode: OutputMode) -> String; } +/// A single line in the rendered spec, with optional scope and children +#[pyclass(get_all, set_all)] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RenderedSpecLine { + /// The formatted content of the line (e.g., "Import: name=documents, source=LocalFile") + pub content: String, + /// The scope name, if applicable (e.g., "documents_1" for ForEach scopes) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scope: Option, + /// Child lines in the hierarchy + pub children: Vec, +} + +/// A rendered specification, grouped by sections +#[pyclass(get_all, set_all)] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RenderedSpec { + /// List of (section_name, lines) pairs + pub sections: Vec<(String, Vec)>, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "kind")] pub enum SpecString { @@ -190,7 +200,7 @@ impl ValueMapping { } impl std::fmt::Display for ValueMapping { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result { match self { ValueMapping::Constant(v) => write!( f, diff --git a/src/py/mod.rs b/src/py/mod.rs index ca33503e..6e582d6f 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -3,7 +3,7 @@ use crate::prelude::*; use crate::base::schema::{FieldSchema, ValueType}; use crate::base::spec::VectorSimilarityMetric; use crate::base::spec::{NamedSpec, ReactiveOpSpec}; -use crate::base::spec::{OutputMode, SpecFormatter}; +use crate::base::spec::{OutputMode, RenderedSpec, RenderedSpecLine, SpecFormatter}; use crate::execution::query; use crate::lib_context::{clear_lib_context, get_auth_registry, init_lib_context}; use crate::ops::interface::{QueryResult, QueryResults}; @@ -198,76 +198,94 @@ impl Flow { }) } - #[pyo3(signature = (output_mode="concise"))] - pub fn get_spec(&self, output_mode: &str) -> PyResult> { - let mode = OutputMode::from_str(output_mode); + #[pyo3(signature = (output_mode=None))] + pub fn get_spec(&self, output_mode: Option>) -> PyResult { + let mode = output_mode.map_or(OutputMode::Concise, |m| m.into_inner()); let spec = &self.0.flow.flow_instance; - let mut result = Vec::with_capacity( - 1 + spec.import_ops.len() - + spec.reactive_ops.len() - + spec.export_ops.len() - + spec.declarations.len(), + let mut sections: IndexMap> = IndexMap::new(); + + // Initialize sections + sections.insert( + "Header".to_string(), + vec![RenderedSpecLine { + content: format!("Flow: {}", spec.name), + scope: None, + children: vec![], + }], ); - - fn extend_with( - out: &mut Vec<(String, String, u32)>, - label: &'static str, - items: I, - f: F, - ) where - I: IntoIterator, - F: Fn(&T) -> String, - { - out.extend(items.into_iter().map(|item| (label.into(), f(&item), 0))); + for key in ["Sources", "Processing", "Targets", "Declarations"] { + sections.insert(key.to_string(), Vec::new()); } - // Header - result.push(("Header".into(), format!("Flow: {}", spec.name), 0)); - // Sources - extend_with(&mut result, "Sources", spec.import_ops.iter(), |op| { - format!("Import: name={}, {}", op.name, op.spec.format(mode)) - }); + for op in &spec.import_ops { + sections + .entry("Sources".to_string()) + .or_default() + .push(RenderedSpecLine { + content: format!("Import: name={}, {}", op.name, op.spec.format(mode)), + scope: None, + children: vec![], + }); + } // Processing fn walk( op: &NamedSpec, - indent: u32, mode: OutputMode, - out: &mut Vec<(String, String, u32)>, - ) { - out.push(( - "Processing".into(), - format!("{}: {}", op.name, op.spec.format(mode)), - indent, - )); + scope: Option, + ) -> RenderedSpecLine { + let content = format!("{}: {}", op.name, op.spec.format(mode)); + let mut line = RenderedSpecLine { + content, + scope, + children: vec![], + }; if let ReactiveOpSpec::ForEach(fe) = &op.spec { - out.push(("Processing".into(), fe.op_scope.to_string(), indent + 1)); for nested in &fe.op_scope.ops { - walk(nested, indent + 2, mode, out); + line.children + .push(walk(nested, mode, Some(fe.op_scope.name.clone()))); } } + + line } for op in &spec.reactive_ops { - walk(op, 0, mode, &mut result); + sections + .entry("Processing".to_string()) + .or_default() + .push(walk(op, mode, None)); } // Targets - extend_with(&mut result, "Targets", spec.export_ops.iter(), |op| { - format!("Export: name={}, {}", op.name, op.spec.format(mode)) - }); + for op in &spec.export_ops { + sections + .entry("Targets".to_string()) + .or_default() + .push(RenderedSpecLine { + content: format!("Export: name={}, {}", op.name, op.spec.format(mode)), + scope: None, + children: vec![], + }); + } // Declarations - extend_with( - &mut result, - "Declarations", - spec.declarations.iter(), - |decl| format!("Declaration: {}", decl.format(mode)), - ); + for decl in &spec.declarations { + sections + .entry("Declarations".to_string()) + .or_default() + .push(RenderedSpecLine { + content: format!("Declaration: {}", decl.format(mode)), + scope: None, + children: vec![], + }); + } - Ok(result) + Ok(RenderedSpec { + sections: sections.into_iter().collect(), + }) } pub fn get_schema(&self) -> Vec<(String, String, String)> { @@ -518,6 +536,8 @@ fn cocoindex_engine(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; Ok(()) } From 15897e931461a4b7466799c902b9f91f488d7177 Mon Sep 17 00:00:00 2001 From: lemorage Date: Sat, 10 May 2025 09:32:37 +0200 Subject: [PATCH 7/8] feat: use bulk load to collect rendered spec line --- src/py/mod.rs | 95 ++++++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 51 deletions(-) diff --git a/src/py/mod.rs b/src/py/mod.rs index 6e582d6f..bedb8f34 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -204,30 +204,18 @@ impl Flow { let spec = &self.0.flow.flow_instance; let mut sections: IndexMap> = IndexMap::new(); - // Initialize sections - sections.insert( - "Header".to_string(), - vec![RenderedSpecLine { - content: format!("Flow: {}", spec.name), - scope: None, - children: vec![], - }], - ); - for key in ["Sources", "Processing", "Targets", "Declarations"] { - sections.insert(key.to_string(), Vec::new()); - } - // Sources - for op in &spec.import_ops { - sections - .entry("Sources".to_string()) - .or_default() - .push(RenderedSpecLine { + sections.insert( + "Source".to_string(), + spec.import_ops + .iter() + .map(|op| RenderedSpecLine { content: format!("Import: name={}, {}", op.name, op.spec.format(mode)), scope: None, children: vec![], - }); - } + }) + .collect(), + ); // Processing fn walk( @@ -236,52 +224,57 @@ impl Flow { scope: Option, ) -> RenderedSpecLine { let content = format!("{}: {}", op.name, op.spec.format(mode)); - let mut line = RenderedSpecLine { - content, - scope, - children: vec![], + + let children = match &op.spec { + ReactiveOpSpec::ForEach(fe) => fe + .op_scope + .ops + .iter() + .map(|nested| walk(nested, mode, Some(fe.op_scope.name.clone()))) + .collect(), + _ => vec![], }; - if let ReactiveOpSpec::ForEach(fe) = &op.spec { - for nested in &fe.op_scope.ops { - line.children - .push(walk(nested, mode, Some(fe.op_scope.name.clone()))); - } + RenderedSpecLine { + content, + scope, + children, } - - line } - for op in &spec.reactive_ops { - sections - .entry("Processing".to_string()) - .or_default() - .push(walk(op, mode, None)); - } + sections.insert( + "Processing".to_string(), + spec.reactive_ops + .iter() + .map(|op| walk(op, mode, None)) + .collect(), + ); // Targets - for op in &spec.export_ops { - sections - .entry("Targets".to_string()) - .or_default() - .push(RenderedSpecLine { + sections.insert( + "Targets".to_string(), + spec.export_ops + .iter() + .map(|op| RenderedSpecLine { content: format!("Export: name={}, {}", op.name, op.spec.format(mode)), scope: None, children: vec![], - }); - } + }) + .collect(), + ); // Declarations - for decl in &spec.declarations { - sections - .entry("Declarations".to_string()) - .or_default() - .push(RenderedSpecLine { + sections.insert( + "Declarations".to_string(), + spec.declarations + .iter() + .map(|decl| RenderedSpecLine { content: format!("Declaration: {}", decl.format(mode)), scope: None, children: vec![], - }); - } + }) + .collect(), + ); Ok(RenderedSpec { sections: sections.into_iter().collect(), From f346d9ac87524e998afdadacfef0280881d76a0b Mon Sep 17 00:00:00 2001 From: lemorage Date: Sat, 10 May 2025 09:52:06 +0200 Subject: [PATCH 8/8] refactor: move rendered spec related struct into pymod --- src/base/spec.rs | 22 ---------------------- src/py/mod.rs | 43 +++++++++++++++++++++++-------------------- 2 files changed, 23 insertions(+), 42 deletions(-) diff --git a/src/base/spec.rs b/src/base/spec.rs index 7c75f960..6a426908 100644 --- a/src/base/spec.rs +++ b/src/base/spec.rs @@ -1,7 +1,6 @@ use crate::prelude::*; use super::schema::{EnrichedValueType, FieldSchema}; -use pyo3::prelude::*; use serde::{Deserialize, Serialize}; use std::fmt; use std::ops::Deref; @@ -19,27 +18,6 @@ pub trait SpecFormatter { fn format(&self, mode: OutputMode) -> String; } -/// A single line in the rendered spec, with optional scope and children -#[pyclass(get_all, set_all)] -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RenderedSpecLine { - /// The formatted content of the line (e.g., "Import: name=documents, source=LocalFile") - pub content: String, - /// The scope name, if applicable (e.g., "documents_1" for ForEach scopes) - #[serde(default, skip_serializing_if = "Option::is_none")] - pub scope: Option, - /// Child lines in the hierarchy - pub children: Vec, -} - -/// A rendered specification, grouped by sections -#[pyclass(get_all, set_all)] -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RenderedSpec { - /// List of (section_name, lines) pairs - pub sections: Vec<(String, Vec)>, -} - #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "kind")] pub enum SpecString { diff --git a/src/py/mod.rs b/src/py/mod.rs index bedb8f34..3c09c99f 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -2,8 +2,7 @@ use crate::prelude::*; use crate::base::schema::{FieldSchema, ValueType}; use crate::base::spec::VectorSimilarityMetric; -use crate::base::spec::{NamedSpec, ReactiveOpSpec}; -use crate::base::spec::{OutputMode, RenderedSpec, RenderedSpecLine, SpecFormatter}; +use crate::base::spec::{NamedSpec, OutputMode, ReactiveOpSpec, SpecFormatter}; use crate::execution::query; use crate::lib_context::{clear_lib_context, get_auth_registry, init_lib_context}; use crate::ops::interface::{QueryResult, QueryResults}; @@ -115,6 +114,24 @@ impl IndexUpdateInfo { #[pyclass] pub struct Flow(pub Arc); +/// A single line in the rendered spec, with hierarchical children +#[pyclass(get_all, set_all)] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RenderedSpecLine { + /// The formatted content of the line (e.g., "Import: name=documents, source=LocalFile") + pub content: String, + /// Child lines in the hierarchy + pub children: Vec, +} + +/// A rendered specification, grouped by sections +#[pyclass(get_all, set_all)] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RenderedSpec { + /// List of (section_name, lines) pairs + pub sections: Vec<(String, Vec)>, +} + #[pyclass] pub struct FlowLiveUpdater(pub Arc>); @@ -211,18 +228,13 @@ impl Flow { .iter() .map(|op| RenderedSpecLine { content: format!("Import: name={}, {}", op.name, op.spec.format(mode)), - scope: None, children: vec![], }) .collect(), ); // Processing - fn walk( - op: &NamedSpec, - mode: OutputMode, - scope: Option, - ) -> RenderedSpecLine { + fn walk(op: &NamedSpec, mode: OutputMode) -> RenderedSpecLine { let content = format!("{}: {}", op.name, op.spec.format(mode)); let children = match &op.spec { @@ -230,24 +242,17 @@ impl Flow { .op_scope .ops .iter() - .map(|nested| walk(nested, mode, Some(fe.op_scope.name.clone()))) + .map(|nested| walk(nested, mode)) .collect(), _ => vec![], }; - RenderedSpecLine { - content, - scope, - children, - } + RenderedSpecLine { content, children } } sections.insert( "Processing".to_string(), - spec.reactive_ops - .iter() - .map(|op| walk(op, mode, None)) - .collect(), + spec.reactive_ops.iter().map(|op| walk(op, mode)).collect(), ); // Targets @@ -257,7 +262,6 @@ impl Flow { .iter() .map(|op| RenderedSpecLine { content: format!("Export: name={}, {}", op.name, op.spec.format(mode)), - scope: None, children: vec![], }) .collect(), @@ -270,7 +274,6 @@ impl Flow { .iter() .map(|decl| RenderedSpecLine { content: format!("Declaration: {}", decl.format(mode)), - scope: None, children: vec![], }) .collect(),