[feat] DuckDb plugin: drag and drop file directly as table (#2952)

- DuckDB plugin: drag and drop file directly as table --------- Signed-off-by: Ihor Dykhta <dikhta.igor@gmail.com>
keplergl · Feb 28, 2025 · 9762dc3 · 9762dc3
1 parent 8e737e8
commit 9762dc3
Show file tree

Hide file tree

Showing 3 changed files with 261 additions and 14 deletions.
diff --git a/src/duckdb/src/components/schema-panel.tsx b/src/duckdb/src/components/schema-panel.tsx
@@ -6,6 +6,7 @@ import {useSelector} from 'react-redux';
 import styled from 'styled-components';
 import {AsyncDuckDBConnection} from '@duckdb/duckdb-wasm';
 
+import {LoadingSpinner, Icons} from '@kepler.gl/components';
 import {arrowSchemaToFields} from '@kepler.gl/processors';
 import {VisState} from '@kepler.gl/schemas';
 
@@ -29,6 +30,14 @@ const StyledSchemaPanel = styled.div`
   font-size: 12px;
   padding: 12px;
   font-family: ${props => props.theme.fontFamily};
+  height: 100%;
+`;
+
+const StyledLoadingSpinnerWrapper = styled.div`
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  height: 100%;
 `;
 
 async function getColumnSchema(connection: AsyncDuckDBConnection, tableName: string) {
@@ -57,16 +66,61 @@ async function getColumnSchema(connection: AsyncDuckDBConnection, tableName: str
   };
 }
 
-function getSchemaSuggestion(result) {
+export type SchemaSuggestion = {column_name: string; table_name: string};
+
+function getSchemaSuggestion(result: {key: string; children: {key: string}[]}[]) {
   return result.reduce((accu, data) => {
     const columns = data.children.map(child => ({
       column_name: child.key,
       table_name: data.key
     }));
     return accu.concat(columns);
-  }, []);
+  }, [] as SchemaSuggestion[]);
 }
-export const SchemaPanel = ({setTableSchema}) => {
+
+type SchemaPanelProps = {
+  setTableSchema: (tableSchema: SchemaSuggestion[]) => void;
+  droppedFile: File | null;
+};
+
+const StyledSchemaPanelDropMessage = styled.div`
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  height: 100%;
+  flex-direction: column;
+  text-align: center;
+
+  div {
+    margin: 5px;
+  }
+  .header {
+    font-size: 15px;
+  }
+  .bold {
+    font-weight: 700;
+  }
+`;
+
+const StyledAddIcon = styled(Icons.Add)`
+  display: inline;
+  margin-top: -3px;
+`;
+
+export const SchemaPanelDropMessage = () => {
+  return (
+    <StyledSchemaPanelDropMessage>
+      <div className="header">
+        <StyledAddIcon /> Add files to DuckDB
+      </div>
+      <div className="bold">Supported formats: </div>
+      <div>.csv, .json, .geojson, .parquet, .arrow</div>
+      <div>Files you add will stay local to your browser.</div>
+    </StyledSchemaPanelDropMessage>
+  );
+};
+
+export const SchemaPanel = ({setTableSchema, droppedFile}: SchemaPanelProps) => {
   const [columnSchemas, setColumnSchemas] = useState<TreeNodeData<{type: string}>[]>([]);
   const datasets = useSelector((state: State) => state?.demo?.keplerGl?.map?.visState.datasets);
 
@@ -76,7 +130,7 @@ export const SchemaPanel = ({setTableSchema}) => {
 
     const tableResult = await c.query('SHOW TABLES;');
 
-    const tableNames = tableResult.getChildAt(0)?.toJSON();
+    const tableNames: string[] | undefined = tableResult.getChildAt(0)?.toJSON();
 
     const result = await Promise.all((tableNames || [])?.map(name => getColumnSchema(c, name)));
     const tableSchema = getSchemaSuggestion(result);
@@ -88,7 +142,7 @@ export const SchemaPanel = ({setTableSchema}) => {
 
   useEffect(() => {
     getTableSchema();
-  }, [datasets, getTableSchema]);
+  }, [datasets, droppedFile, getTableSchema]);
 
   return (
     <StyledSchemaPanel>
@@ -107,8 +161,12 @@ export const SchemaPanel = ({setTableSchema}) => {
             }}
           />
         ))
+      ) : droppedFile ? (
+        <StyledLoadingSpinnerWrapper>
+          <LoadingSpinner />
+        </StyledLoadingSpinnerWrapper>
       ) : (
-        <div>No tables found</div>
+        <SchemaPanelDropMessage />
       )}
     </StyledSchemaPanel>
   );

diff --git a/src/duckdb/src/components/sql-panel.tsx b/src/duckdb/src/components/sql-panel.tsx
@@ -2,20 +2,20 @@
 // Copyright contributors to the kepler.gl project
 
 import * as arrow from 'apache-arrow';
-import React, {useCallback, useState, useEffect} from 'react';
+import React, {useCallback, useState, useEffect, useRef} from 'react';
 import {useDispatch} from 'react-redux';
 import styled from 'styled-components';
 import {Panel, PanelGroup, PanelResizeHandle} from 'react-resizable-panels';
 
 import {addDataToMap} from '@kepler.gl/actions';
 import {generateHashId} from '@kepler.gl/common-utils';
-import {Button, IconButton, Icons, LoadingSpinner, Tooltip} from '@kepler.gl/components';
+import {Button, FileDrop, IconButton, Icons, LoadingSpinner, Tooltip} from '@kepler.gl/components';
 import {arrowSchemaToFields} from '@kepler.gl/processors';
 import {sidePanelBg, panelBorderColor} from '@kepler.gl/styles';
 import {isAppleDevice} from '@kepler.gl/utils';
 
 import MonacoEditor from './monaco-editor';
-import {SchemaPanel} from './schema-panel';
+import {SchemaPanel, SchemaSuggestion} from './schema-panel';
 import {PreviewDataPanel, QueryResult} from './preview-data-panel';
 import {getDuckDB} from '../init';
 import {
@@ -26,7 +26,9 @@ import {
   setGeoArrowWKBExtension,
   splitSqlStatements,
   checkIsSelectQuery,
-  removeSQLComments
+  removeSQLComments,
+  tableFromFile,
+  SUPPORTED_DUCKDB_DROP_EXTENSIONS
 } from '../table/duckdb-table-utils';
 
 const StyledSqlPanel = styled.div`
@@ -125,6 +127,20 @@ const StyledErrorContainer = styled.pre`
   overflow: auto;
 `;
 
+interface StyledDragPanelProps {
+  dragOver?: boolean;
+}
+
+const StyledFileDropArea = styled(FileDrop)<StyledDragPanelProps>`
+  height: 100%;
+  border-width: 1px;
+  border: 1px ${props => (props.dragOver ? 'solid' : 'dashed')}
+    ${props => (props.dragOver ? props.theme.subtextColorLT : 'transparent')};
+  .file-drop-target {
+    height: 100%;
+  }
+`;
+
 type SqlPanelProps = {
   initialSql?: string;
 };
@@ -136,14 +152,18 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
     const params = new URLSearchParams(window.location.search);
     return params.get('sql') || initialSql;
   });
+  const [droppedFile, setDroppedFile] = useState<File | null>(null);
+  const [dragState, setDragState] = useState(false);
   const [result, setResult] = useState<null | QueryResult>(null);
   const [error, setError] = useState<Error | null>(null);
   const [counter, setCounter] = useState(0);
-  const [tableSchema, setTableSchema] = useState([]);
+  const [tableSchema, setTableSchema] = useState<SchemaSuggestion[]>([]);
   const [isRunning, setIsRunning] = useState(false);
   const [isMac] = useState(() => isAppleDevice());
   const dispatch = useDispatch();
 
+  const droppedFileAreaRef = useRef(null);
+
   useEffect(() => {
     const currentUrl = new URL(window.location.href);
     if (sql) {
@@ -244,11 +264,73 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
     setCounter(counter + 1);
   }, [result, counter, dispatch]);
 
+  const isValidFileType = useCallback(filename => {
+    const fileExt = SUPPORTED_DUCKDB_DROP_EXTENSIONS.find(ext => filename.endsWith(ext));
+    return Boolean(fileExt);
+  }, []);
+
+  const createTableFromDroppedFile = useCallback(async (droppedFile: File | null) => {
+    if (droppedFile) {
+      const error = await tableFromFile(droppedFile);
+      if (error) {
+        setError(error);
+      } else {
+        setError(null);
+      }
+    }
+
+    setDroppedFile(null);
+    setDragState(false);
+  }, []);
+
+  useEffect(() => {
+    createTableFromDroppedFile(droppedFile);
+  }, [droppedFile, createTableFromDroppedFile]);
+
+  const handleFileInput = useCallback(
+    (fileList: FileList, event: DragEvent) => {
+      if (event) {
+        event.preventDefault();
+        event.stopPropagation();
+      }
+
+      const files = [...fileList].filter(Boolean);
+
+      const disableExtensionFilter = false;
+
+      const filesToLoad: File[] = [];
+      const errorFiles: string[] = [];
+      for (const file of files) {
+        if (disableExtensionFilter || isValidFileType(file.name)) {
+          filesToLoad.push(file);
+        } else {
+          errorFiles.push(file.name);
+        }
+      }
+
+      if (filesToLoad.length > 0) {
+        setDroppedFile(filesToLoad[0]);
+      } else if (errorFiles.length > 0) {
+        setError(new Error(`Unsupported file formats: ${errorFiles.join(', ')}`));
+      }
+    },
+    [isValidFileType]
+  );
+
   return (
     <StyledSqlPanel>
       <PanelGroup direction="horizontal">
         <Panel defaultSize={20} minSize={15} style={SCHEMA_PANEL_STYLE}>
-          <SchemaPanel setTableSchema={setTableSchema} />
+          <StyledFileDropArea
+            dragOver={dragState}
+            onDragOver={() => setDragState(true)}
+            onDragLeave={() => setDragState(false)}
+            frame={droppedFileAreaRef.current || document}
+            onDrop={handleFileInput}
+            className="file-uploader__file-drop"
+          >
+            <SchemaPanel setTableSchema={setTableSchema} droppedFile={droppedFile} />
+          </StyledFileDropArea>
         </Panel>
 
         <StyledResizeHandle />
@@ -284,7 +366,7 @@ export const SqlPanel: React.FC<SqlPanelProps> = ({initialSql = ''}) => {
               </Panel>
 
               <StyledVerticalResizeHandle />
-              <Panel>
+              <Panel className="preview-panel">
                 {isRunning ? (
                   <StyledLoadingContainer>
                     <LoadingSpinner />