From 44526ebcd9949da907dfb5a67e44fad2409bba71 Mon Sep 17 00:00:00 2001 From: Xun Li Date: Tue, 23 Apr 2024 09:24:55 -0700 Subject: [PATCH] [Feat] Kepler-Jupyter 0.3.4 with kepler v3 (#2565) --- .github/workflows/build-publish-pypi.yml | 4 +- .gitignore | 3 +- bindings/kepler.gl-jupyter/README.md | 16 +- bindings/kepler.gl-jupyter/RELEASE.md | 6 +- bindings/kepler.gl-jupyter/js/babel.config.js | 2 +- .../js/lib/keplergl/components/app.js | 2 +- .../lib/keplergl/components/config-panel.js | 6 +- .../lib/keplergl/components/panel-header.js | 4 +- .../js/lib/keplergl/components/side-bar.js | 2 +- .../js/lib/keplergl/kepler.gl.js | 7 +- .../js/lib/keplergl/store.js | 5 +- .../js/lib/keplergl/utils.js | 18 +- bindings/kepler.gl-jupyter/js/package.json | 35 +++- .../js/webpack/build-html.js | 9 +- .../kepler.gl-jupyter/js/webpack/config.js | 29 ++- .../kepler.gl-jupyter/keplergl/_version.py | 4 +- .../kepler.gl-jupyter/keplergl/keplergl.py | 167 ++++++++++++++---- bindings/kepler.gl-jupyter/pyproject.toml | 4 +- bindings/kepler.gl-jupyter/requirements.txt | 11 +- bindings/kepler.gl-jupyter/setup.py | 15 +- website/webpack.config.js | 3 +- 21 files changed, 255 insertions(+), 97 deletions(-) diff --git a/.github/workflows/build-publish-pypi.yml b/.github/workflows/build-publish-pypi.yml index 15dde697c3..bd84d4167a 100644 --- a/.github/workflows/build-publish-pypi.yml +++ b/.github/workflows/build-publish-pypi.yml @@ -16,10 +16,10 @@ jobs: node-version: 18.x registry-url: https://registry.npmjs.org/ - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Install dependencies run: | diff --git a/.gitignore b/.gitignore index c30aa0683e..9fdd266774 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ package-lock.json .DS_Store .idea .vscode/ +.venv/ /bundle*.js /favicon.png @@ -33,4 +34,4 @@ npm-debug.log !.yarn/plugins !.yarn/releases !.yarn/sdks -!.yarn/versions \ No newline at end of file +!.yarn/versions diff --git a/bindings/kepler.gl-jupyter/README.md b/bindings/kepler.gl-jupyter/README.md index 54e69c0aeb..eb60b450e0 100644 --- a/bindings/kepler.gl-jupyter/README.md +++ b/bindings/kepler.gl-jupyter/README.md @@ -47,7 +47,7 @@ jupyter nbextension enable --py --sys-prefix keplergl # can be skipped for noteb ### 2. For Google Colab: -`keplergl` (>0.3.0) works with Google Colab. You can install it using pip. +`keplergl` (>0.3.0) works with Google Colab. You can install it using pip. ```python # Install keplergl (>0.3.0) @@ -58,7 +58,7 @@ jupyter nbextension enable --py --sys-prefix keplergl # can be skipped for noteb #### JupyterLab 3 -NOTE: `keplergl` <=0.3.0 doesn't work with JupyterLab 3. You need to make sure the python package `keplergl` > 0.3.0 is installed. +NOTE: `keplergl` <=0.3.0 doesn't work with JupyterLab 3. You need to make sure the python package `keplergl` > 0.3.0 is installed. Installation using pip: ```shell @@ -73,7 +73,7 @@ There is no need to use `jupyter labextension install` for `keplergl` > 0.3.0 wi #### JupyterLab 1 -For JupyterLab1, you need to install `keplergl-jupyter` labextension from NPM registery. There is no need to install `keplergl` python package. +For JupyterLab1, you need to install `keplergl-jupyter` labextension from NPM registery. There is no need to install `keplergl` python package. First, install `jupyterlab-manager` for JupyterLab1: ```shell @@ -91,7 +91,7 @@ jupyter labextension install keplergl-jupyter #### JupyterLab 2 -For JupyterLab2, you need to install `keplergl-jupyter` labextension from NPM registery. There is no need to install `keplergl` python package. +For JupyterLab2, you need to install `keplergl-jupyter` labextension from NPM registery. There is no need to install `keplergl` python package. First, install `jupyterlab-manager` for JupyterLab2: ```shell @@ -127,10 +127,12 @@ map_1 = KeplerGl(height=400) map_1 # Load kepler.gl with map data and config +# Since keplergl 0.3.4, you can pass `use_arrow=True` to load and render data faster using GeoArrow, e.g. `KeplerGl(data={'data_1': df}, config=config, use_arrow=True)` map_2 = KeplerGl(height=400, data={'data_1': df}, config=config) map_2 # Add data to map +# Since keplergl 0.3.4, you can pass `use_arrow=True` to load and render data faster using GeoArrow, e.g. `map_1.add_data(df, 'data_1', use_arrow=True)` map_1.add_data(df, 'data_1') # Apply config @@ -209,17 +211,19 @@ You will need to install node, yarn and Jupyter Notebook. Install [node](https://nodejs.org/en/download/package-manager/#macos) `> 12`, and [yarn](https://yarnpkg.com/en/docs/install#mac-stable). Use [nvm](https://github.com/creationix/nvm) for better node version management e.g. `nvm install 12`. -#### 2. Install Jupyter +#### 2. Install Jupyter - Using conda ```shell conda install jupyter +conda install notebook 6.0.1 ``` - Using pip ```shell pip install jupyter +pip install notebook==6.0.1 ``` #### 3. Install GeoPandas @@ -281,7 +285,7 @@ jupyter nbextension install --py --symlink --sys-prefix keplergl jupyter nbextension enable --py --sys-prefix keplergl ``` -NOTE: The above command `jupyter nbextension install -py --symlink --sys-prefix keplergl` is trying to create a symoblic link of the folder `bindings/kepler.gl-jupyter/keplergl/static` under the jupyter's folder `nbextensions`. Please check if there is already a folder "nbextensions/kepler-jupyter" existed, and you might need to remove it first. +NOTE: The above command `jupyter nbextension install -py --symlink --sys-prefix keplergl` is trying to create a symoblic link of the folder `bindings/kepler.gl-jupyter/keplergl/static` under the jupyter's folder `nbextensions`. Please check if there is already a folder "nbextensions/kepler-jupyter" existed, and you might need to remove it first. To find the location of `nbextensions` folder, you can use the following command: ```shell diff --git a/bindings/kepler.gl-jupyter/RELEASE.md b/bindings/kepler.gl-jupyter/RELEASE.md index ea522a10cf..10c96577cd 100644 --- a/bindings/kepler.gl-jupyter/RELEASE.md +++ b/bindings/kepler.gl-jupyter/RELEASE.md @@ -8,7 +8,7 @@ NOTE: __Version number of the js module **`kelergl-jupyter`** and the python mod ### Step1: -Update `version_info` in keplergl/_version.py in bindings/kepler.gl-jupyter folder. +Update `version_info` in keplergl/_version.py in bindings/kepler.gl-jupyter folder. Update `"version": "0.x.x"` to match the version info in js/package.json in bindings/kepler.gl-jupyter folder. Update `EXTENSION_SPEC_VERSION` to match the js module version. Update `version` in js/package @@ -21,7 +21,7 @@ git commit -am "keplergl==" ### Step2: -Create a tag: `-jupyter` e.g. v0.3.2-jupyter +Create a tag: `-jupyter` e.g. v0.3.4-jupyter ``` git tag -a -jupyter -m "-jupyter" @@ -38,7 +38,7 @@ The new version should be automatically picked and built from PyPi by conda-forg Edit `meta.yaml` under directory `recipes/`: -* Update the version number +* Update the version number ```python {% set version = "0.3.0" %} diff --git a/bindings/kepler.gl-jupyter/js/babel.config.js b/bindings/kepler.gl-jupyter/js/babel.config.js index f5c556f3d6..c0558b8329 100644 --- a/bindings/kepler.gl-jupyter/js/babel.config.js +++ b/bindings/kepler.gl-jupyter/js/babel.config.js @@ -6,7 +6,7 @@ const KeplerPackage = require('./package'); module.exports = function babel(api) { api.cache(true); - const presets = ['@babel/preset-env', '@babel/preset-react']; + const presets = ['@babel/preset-env', '@babel/preset-react', '@babel/preset-typescript']; const plugins = [ '@babel/plugin-proposal-class-properties', 'transform-inline-environment-variables', diff --git a/bindings/kepler.gl-jupyter/js/lib/keplergl/components/app.js b/bindings/kepler.gl-jupyter/js/lib/keplergl/components/app.js index 327f5c1e6d..a10b6c1688 100644 --- a/bindings/kepler.gl-jupyter/js/lib/keplergl/components/app.js +++ b/bindings/kepler.gl-jupyter/js/lib/keplergl/components/app.js @@ -12,7 +12,7 @@ import { PanelHeaderFactory, CustomPanelsFactory, injectComponents -} from 'kepler.gl/components'; +} from '@kepler.gl/components'; import CustomPanelHeaderFactory from './panel-header'; import CustomSidebarFactory from './side-bar'; diff --git a/bindings/kepler.gl-jupyter/js/lib/keplergl/components/config-panel.js b/bindings/kepler.gl-jupyter/js/lib/keplergl/components/config-panel.js index dca25fa040..11b6494fc3 100644 --- a/bindings/kepler.gl-jupyter/js/lib/keplergl/components/config-panel.js +++ b/bindings/kepler.gl-jupyter/js/lib/keplergl/components/config-panel.js @@ -3,9 +3,9 @@ import React, {useState} from 'react'; -import {Button, Icons, TextArea, withState} from 'kepler.gl/components'; -import {KeplerGlSchema} from 'kepler.gl/schemas'; -import {visStateLens, mapStateLens, mapStyleLens} from 'kepler.gl/reducers'; +import {Button, Icons, TextArea, withState} from '@kepler.gl/components'; +import {KeplerGlSchema} from '@kepler.gl/schemas'; +import {visStateLens, mapStateLens, mapStyleLens} from '@kepler.gl/reducers'; import {CopyToClipboard} from 'react-copy-to-clipboard'; import styled from 'styled-components'; diff --git a/bindings/kepler.gl-jupyter/js/lib/keplergl/components/panel-header.js b/bindings/kepler.gl-jupyter/js/lib/keplergl/components/panel-header.js index 886bac8a67..26668d7e4a 100644 --- a/bindings/kepler.gl-jupyter/js/lib/keplergl/components/panel-header.js +++ b/bindings/kepler.gl-jupyter/js/lib/keplergl/components/panel-header.js @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MIT // Copyright contributors to the kepler.gl project -import {PanelHeaderFactory, Icons, withState} from 'kepler.gl/components'; -import {toggleModal} from 'kepler.gl/actions'; +import {PanelHeaderFactory, Icons, withState} from '@kepler.gl/components'; +import {toggleModal} from '@kepler.gl/actions'; import React from 'react'; import {IntlProvider} from 'react-intl'; diff --git a/bindings/kepler.gl-jupyter/js/lib/keplergl/components/side-bar.js b/bindings/kepler.gl-jupyter/js/lib/keplergl/components/side-bar.js index d541c98402..0a8116044b 100644 --- a/bindings/kepler.gl-jupyter/js/lib/keplergl/components/side-bar.js +++ b/bindings/kepler.gl-jupyter/js/lib/keplergl/components/side-bar.js @@ -2,7 +2,7 @@ // Copyright contributors to the kepler.gl project import React from 'react'; -import {SidebarFactory, CollapseButtonFactory} from 'kepler.gl/components'; +import {SidebarFactory, CollapseButtonFactory} from '@kepler.gl/components'; import styled from 'styled-components'; const StyledSideBarContainer = styled.div` diff --git a/bindings/kepler.gl-jupyter/js/lib/keplergl/kepler.gl.js b/bindings/kepler.gl-jupyter/js/lib/keplergl/kepler.gl.js index cc4c4612f7..b80865d22f 100644 --- a/bindings/kepler.gl-jupyter/js/lib/keplergl/kepler.gl.js +++ b/bindings/kepler.gl-jupyter/js/lib/keplergl/kepler.gl.js @@ -1,8 +1,8 @@ // SPDX-License-Identifier: MIT // Copyright contributors to the kepler.gl project -import {addDataToMap, ActionTypes} from 'kepler.gl/actions'; -import {KeplerGlSchema} from 'kepler.gl/schemas'; +import {addDataToMap, ActionTypes} from '@kepler.gl/actions'; +import {KeplerGlSchema} from '@kepler.gl/schemas'; import document from 'global/document'; import renderRoot from './components/root'; @@ -142,7 +142,8 @@ class KeplerGlJupyter { }, data: { fields: d.fields, - rows: d.allData + // rows: d.allData + ...(d.dataContainer instanceof ArrowDataContainer ? {cols: d.dataContainer._cols} : {rows: d.allData}) } })), config, diff --git a/bindings/kepler.gl-jupyter/js/lib/keplergl/store.js b/bindings/kepler.gl-jupyter/js/lib/keplergl/store.js index f452776547..d0c72fc679 100644 --- a/bindings/kepler.gl-jupyter/js/lib/keplergl/store.js +++ b/bindings/kepler.gl-jupyter/js/lib/keplergl/store.js @@ -3,10 +3,7 @@ import {applyMiddleware, compose, createStore} from 'redux'; import {combineReducers} from 'redux'; -import {keplerGlReducer} from 'kepler.gl/reducers'; - -// TODO: remove this after added middleware to files -import {enhanceReduxMiddleware} from 'kepler.gl/middleware'; +import {keplerGlReducer, enhanceReduxMiddleware} from '@kepler.gl/reducers'; const customizedKeplerGlReducer = keplerGlReducer.initialState({ uiState: { diff --git a/bindings/kepler.gl-jupyter/js/lib/keplergl/utils.js b/bindings/kepler.gl-jupyter/js/lib/keplergl/utils.js index 0ea98423b1..413c3cddb0 100644 --- a/bindings/kepler.gl-jupyter/js/lib/keplergl/utils.js +++ b/bindings/kepler.gl-jupyter/js/lib/keplergl/utils.js @@ -1,7 +1,8 @@ // SPDX-License-Identifier: MIT // Copyright contributors to the kepler.gl project -import {processCsvData, processGeojson} from 'kepler.gl/processors'; +import {tableFromIPC} from 'apache-arrow'; +import {processCsvData, processGeojson, processArrowBatches} from '@kepler.gl/processors'; import log from '../log'; import console from 'global/console'; @@ -23,11 +24,22 @@ function handleJuptyerDataFormat(dataEntry) { type = 'json'; } } else if (typeof data === 'string') { + // check if js string is json string try { parsed = JSON.parse(data); type = 'json'; } catch (e) { - // assume it is csv + // assume it is base64 string represents arrow table + try { + console.log('parse base64string arrow tabl'); + // convert arrowTable from base64 string to ArrayBuffer + const arrowTableBuffer = Buffer.from(data, 'base64').buffer; + // create arrow table from ArrayBuffer + parsed = tableFromIPC([new Uint8Array(arrowTableBuffer)]); + type = 'arrow'; + } catch (e) { + // now we can assume it is csv + } } } @@ -47,6 +59,8 @@ function processReceivedData({data, info}) { ? processGeojson(data) : info.queryType === 'df' ? processDataFrame(data) + : info.queryType === 'arrow' + ? processArrowBatches(data.batches) : null; } catch (e) { console.log( diff --git a/bindings/kepler.gl-jupyter/js/package.json b/bindings/kepler.gl-jupyter/js/package.json index 6a514da7a8..cd30f788ca 100644 --- a/bindings/kepler.gl-jupyter/js/package.json +++ b/bindings/kepler.gl-jupyter/js/package.json @@ -1,6 +1,6 @@ { "name": "keplergl-jupyter", - "version": "0.3.2", + "version": "0.3.4", "description": "This is a simple jupyter widget for kepler.gl, an advanced geo-spatial visualization tool, to render large-scale interactive maps.", "author": "Shan He", "license": "MIT", @@ -26,19 +26,21 @@ "start": "NODE_ENV=development webpack --config ./webpack/dev.js --mode development --watch --progress", "clean": "rimraf dist/ && rimraf ../keplergl/static/", "cleanall": "npm run clean && rimraf node_modules/", - "prepublish": "yarn build && yarn build:lab", - "build": "npm run clean && npm run build:lab && webpack --config ./webpack/build.js && jupyter labextension build .", - "build:lab": "rimraf babel/ && mkdir babel && babel lib --out-dir babel", + "prepublish": "NODE_OPTIONS=--openssl-legacy-provider yarn build && yarn build:lab", + "build": "NODE_OPTIONS=--openssl-legacy-provider npm run clean && npm run build:lab && webpack --config ./webpack/build.js && jupyter labextension build .", + "build:lab": "NODE_OPTIONS=--openssl-legacy-provider rimraf babel/ && mkdir babel && babel lib --out-dir babel", "test": "echo \"Error: no test specified\" && exit 1", "lint": "eslint lib webpack --fix", "prettier": "prettier --config ./.prettierrc --print-width 80 --single-quote --write lib/**/*.js" }, "devDependencies": { + "apache-arrow": "^13.0.0", "@babel/cli": "7.4.4", "@babel/core": "7.4.5", "@babel/plugin-proposal-class-properties": "^7.3.0", "@babel/preset-env": "^7.0.0", "@babel/preset-react": "^7.0.0", + "@babel/preset-typescript": "^7.16.7", "@jupyterlab/builder": "^4.0.0", "babel-eslint": "^9.0.0", "babel-loader": "^8.0.0", @@ -55,7 +57,7 @@ "react-dev-utils": "^10.2.1", "rimraf": "^2.6.1", "webpack": "^4.29.0", - "webpack-cli": "4.10.x", + "webpack-cli": "^3.2.1", "webpack-dev-middleware": "^3.5.1", "webpack-dev-server": "^3.1.14", "webpack-hot-middleware": "^2.24.3", @@ -64,8 +66,17 @@ "dependencies": { "@jupyter-widgets/base": "^2 || ^3 || ^4.0.0", "@jupyterlab/builder": "^4.0.0", + "@kepler.gl/actions": "^3.0.0", + "@kepler.gl/components": "^3.0.0", + "@kepler.gl/processors": "^3.0.0", + "@kepler.gl/reducers": "^3.0.0", + "@kepler.gl/styles": "^3.0.0", + "@kepler.gl/schemas": "^3.0.0", + "@loaders.gl/arrow": "^4.1.0", + "@loaders.gl/core": "^4.1.0", + "@loaders.gl/csv": "^4.1.0", + "@loaders.gl/json": "^4.1.0", "global": "^4.3.0", - "kepler.gl": "^2.5.5", "node-polyfill-webpack-plugin": "^1.1.2", "querystring": "0.2.1", "react": "^18.2.0", @@ -79,8 +90,11 @@ "styled-components": "^4.1.3" }, "resolutions": { - "react-vis": "1.11.7", - "webpack-cli": "4.10.x" + "@deck.gl/core": "^8.9.27", + "@deck.gl/react": "^8.9.27", + "@deck.gl/aggregation-layers": "^8.9.27", + "@deck.gl/geo-layers": "^8.9.27", + "@deck.gl/layers": "^8.9.27" }, "jupyterlab": { "extension": "babel/labplugin", @@ -91,5 +105,10 @@ "singleton": true } } + }, + "packageManager": "yarn@1.22.17", + "volta": { + "node": "18.18.2", + "yarn": "1.22.17" } } diff --git a/bindings/kepler.gl-jupyter/js/webpack/build-html.js b/bindings/kepler.gl-jupyter/js/webpack/build-html.js index 31bdf3bcd6..68546f6ca6 100644 --- a/bindings/kepler.gl-jupyter/js/webpack/build-html.js +++ b/bindings/kepler.gl-jupyter/js/webpack/build-html.js @@ -17,14 +17,15 @@ function clearCarats(version) { const VERSIONS = { react: clearCarats(dependencies.react), - reactDom:clearCarats(dependencies['react-dom']), + reactDom: clearCarats(dependencies['react-dom']), redux: clearCarats(dependencies.redux), reactRedux: clearCarats(dependencies['react-redux']), - reactIntl: clearCarats(dependencies['react-intl']), + // reactIntl UMD build is not available after 5.x + reactIntl: '4.7.6', reactCopyToClipboard: clearCarats(dependencies['react-copy-to-clipboard']), styledComponents: clearCarats(dependencies['styled-components']), - keplergl: clearCarats(dependencies['kepler.gl']) -} + keplergl: clearCarats(dependencies['@kepler.gl/components']) +}; const externals = [ {react: 'React'}, diff --git a/bindings/kepler.gl-jupyter/js/webpack/config.js b/bindings/kepler.gl-jupyter/js/webpack/config.js index 0c35883348..bf2ef5394c 100644 --- a/bindings/kepler.gl-jupyter/js/webpack/config.js +++ b/bindings/kepler.gl-jupyter/js/webpack/config.js @@ -8,10 +8,27 @@ const buildHtml = require('./build-html'); const rules = [ { - test: /\.js$/, + test: /\.(js|jsx|ts|tsx)$/, loader: 'babel-loader', include: path.join(__dirname, '../lib', 'keplergl'), exclude: [/node_modules/] + }, + // fix for arrow-related errors + { + test: /\.mjs$/, + // include: /node_modules\/apache-arrow/, + include: /node_modules/, + type: 'javascript/auto' + }, + // for compiling @probe.gl, website build started to fail (March, 2024) + { + test: /\.(js|ts)$/, + loader: 'babel-loader', + include: [ + /node_modules\/@probe.gl/, + /node_modules\/@loaders.gl/, + /node_modules\/@math.gl/ + ] } ]; @@ -74,19 +91,19 @@ module.exports = { umd: { // Embeddable {{ cookiecutter.npm_package_name }} bundle - + // This bundle is generally almost identical to the notebook bundle // containing the custom widget views and models. - + // The only difference is in the configuration of the webpack public path // for the static assets. - + // It will be automatically distributed by unpkg to work with the static // widget embedder. - + // The target bundle is always `dist/index.js`, which is the path required // by the custom widget embedder. - + entry: path.resolve(__dirname, '../lib/embed.js'), output: { filename: 'index.js', diff --git a/bindings/kepler.gl-jupyter/keplergl/_version.py b/bindings/kepler.gl-jupyter/keplergl/_version.py index 8acca3995f..5faf6b8f39 100644 --- a/bindings/kepler.gl-jupyter/keplergl/_version.py +++ b/bindings/kepler.gl-jupyter/keplergl/_version.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: MIT # Copyright contributors to the kepler.gl project -version_info = (0, 3, 2, 'final', 0) +version_info = (0, 3, 4, 'alpha', 0) _specifier_ = {'alpha': 'a', 'beta': 'b', 'candidate': 'rc', 'final': ''} @@ -15,4 +15,4 @@ # the widget models, or if the serialized format changes. # # The major version needs to match that of the JS package. -EXTENSION_SPEC_VERSION = '^0.3.2' \ No newline at end of file +EXTENSION_SPEC_VERSION = '^0.3.4' diff --git a/bindings/kepler.gl-jupyter/keplergl/keplergl.py b/bindings/kepler.gl-jupyter/keplergl/keplergl.py index ec581935bf..65f9317385 100644 --- a/bindings/kepler.gl-jupyter/keplergl/keplergl.py +++ b/bindings/kepler.gl-jupyter/keplergl/keplergl.py @@ -1,17 +1,40 @@ +"""module for keplergl-jupyter""" # SPDX-License-Identifier: MIT # Copyright contributors to the kepler.gl project +import base64 +import sys +import json import ipywidgets as widgets from pkg_resources import resource_string from traitlets import Unicode, Dict, Int, validate, TraitError import pandas as pd import geopandas +import pyarrow +import geoarrow.pyarrow as ga +import geoarrow.pandas as _ import shapely.wkt -import json from ._version import EXTENSION_SPEC_VERSION -import sys documentation = 'https://docs.kepler.gl/docs/keplergl-jupyter' + +# global variable use_arrow, which can be set by parameter use_arrow in KeplerGl +g_use_arrow = False + +def _arrow_table_to_base64(arrow_table): + '''Convert an arrow table to a base64 string''' + batches = arrow_table.to_batches() + sink = pyarrow.BufferOutputStream() + writer = pyarrow.ipc.new_stream(sink, arrow_table.schema) + for batch in batches: + writer.write_batch(batch) + arrow_buf = sink.getvalue() + + # TODO: we could send the bytes directly to the frontend using traitlets.Bytes + base64_string = base64.b64encode(arrow_buf.to_pybytes()).decode() + + return base64_string + def _df_to_dict(df): ''' Create an input dict for Kepler.gl using a DataFrame object @@ -24,6 +47,21 @@ def _df_to_dict(df): ''' return df.to_dict('split') + +def _df_to_arrow(df: pd.DataFrame): + ''' Create an arrow base64string for Kepler.gl using a DataFrame object + + Inputs: + - df: a DataFrame object + + Returns: + - string: a base64 string that can be used in Kepler.gl + ''' + arrow_table = pyarrow.Table.from_pandas(df) + base64_string = _arrow_table_to_base64(arrow_table) + + return base64_string + def _gdf_to_dict(gdf): ''' Create an input dict for kepler.gl using a GeoDataFrame object @@ -45,49 +83,79 @@ def _gdf_to_dict(gdf): df = pd.DataFrame(gdf) # convert geometry to wkt df[name] = df.geometry.apply(lambda x: shapely.wkt.dumps(x)) + # df[name] = shapely.wkt.dumps(df.geometry) return _df_to_dict(df) -def _normalize_data(data): + +def _gdf_to_arrow(gdf): + ''' Create an arrow base64string for Kepler.gl using a GeoDataFrame object''' + # reproject to 4326 if needed + if gdf.crs and not gdf.crs == 4326: + gdf = gdf.to_crs(4326) + + # get name of the geometry column + # will cause error if data frame has no geometry column + name = gdf.geometry.name + + array = ga.as_geoarrow(gdf.geometry, coord_type=ga.CoordType.INTERLEAVED) + + table = pyarrow.Table.from_pandas(gdf.drop(columns=[name])) + arrow_table = table.append_column(name, array) + base64_string = _arrow_table_to_base64(arrow_table) + + return base64_string + + +def _normalize_data(data, use_arrow=False): if isinstance(data, pd.DataFrame): - return _gdf_to_dict(data) if isinstance(data, geopandas.GeoDataFrame) else _df_to_dict(data) + if use_arrow: + return _gdf_to_arrow(data) if isinstance(data, geopandas.GeoDataFrame) else _df_to_arrow(data) + else: + return _gdf_to_dict(data) if isinstance(data, geopandas.GeoDataFrame) else _df_to_dict(data) return data + def data_to_json(data, manager): - '''Serialize a Python date object. + '''Serialize a Python data object. Attributes of this dictionary are to be passed to the JavaScript side. ''' - if data is None: return None else: - if type(data) is not dict: + if not isinstance(data, dict): print(data) - raise Exception('data type incorrect expecting a dictionary mapping from data id to value, but got {}'.format(type(data))) - return None + raise TraitError( + f"data type incorrect expecting a dictionary mapping from data id to value, but got {type(data)}") else: dataset = {} + # use g_use_arrow to determine if we should use arrow for key, value in data.items(): - normalized = _normalize_data(value) + normalized = _normalize_data(value, g_use_arrow) dataset.update({key: normalized}) return dataset + def data_from_json(js, manager): '''Deserialize a Javascript date.''' return js + data_serialization = { 'from_json': data_from_json, 'to_json': data_to_json } + class TraitError(Exception): pass + class DataException(TraitError): pass + @widgets.register class KeplerGl(widgets.DOMWidget): """An example widget.""" @@ -97,8 +165,9 @@ class KeplerGl(widgets.DOMWidget): _model_module = Unicode('keplergl-jupyter').tag(sync=True) _view_module_version = Unicode(EXTENSION_SPEC_VERSION).tag(sync=True) _model_module_version = Unicode(EXTENSION_SPEC_VERSION).tag(sync=True) - value = Unicode('Hello World!').tag(sync=True) + # Attributes + value = Unicode('Hello World!').tag(sync=True) data = Dict({}).tag(sync=True, **data_serialization) config = Dict({}).tag(sync=True) height = Int(400).tag(sync=True) @@ -107,29 +176,32 @@ def __init__(self, **kwargs): if 'show_docs' not in kwargs: kwargs['show_docs'] = True if kwargs['show_docs']: - print('User Guide: {}'.format(documentation)) + print(f"User Guide: {documentation}") kwargs.pop('show_docs') + # assign use_arrow to global variable + global g_use_arrow + g_use_arrow = kwargs.get('use_arrow', False) + super(KeplerGl, self).__init__(**kwargs) @validate('data') def _validate_data(self, proposal): - '''Validate data input. + '''Validate data input (return from data_to_json) Makes sure data is a dict, and each value should be either a df, a geojson dictionary / string or csv string layers list. ''' - if type(proposal.value) is not dict: - raise DataException('[data type error]: Expecting a dictionary mapping from id to value, but got {}'.format(type(proposal.value))) - + if not isinstance(proposal.value, dict): + raise DataException(f"[data type error]: Expecting a dictionary mapping from id to value, but got {type(proposal.value)}") else: for key, value in proposal.value.items(): - if not isinstance(value, pd.DataFrame) and (type(value) is not str) and (type(value) is not dict): - raise DataException('[data type error]: value of {} should be a DataFrame, a Geojson Dictionary or String, a csv String, but got {}'.format(key, type(value))) + if not isinstance(value, pd.DataFrame) and not isinstance(value, str) and not isinstance(value, dict): + raise DataException(f"[data type error]: value of {key} should be a DataFrame, a Geojson Dictionary or String, a csv String, but got {type(value)}") return proposal.value - def add_data(self, data, name="unnamed"): + def add_data(self, data, name="unnamed", use_arrow=False): ''' Send data to Voyager Inputs: @@ -139,10 +211,20 @@ def add_data(self, data, name="unnamed"): Example of use: keplergl.add_data(data_string, name="data_1") ''' - - normalized = _normalize_data(data) copy = self.data.copy() - copy.update({name: normalized}) + + # assume data is a GeoJSON or CSV string, convert it to arrow if use_arrow is True + if use_arrow: + global g_use_arrow + g_use_arrow = use_arrow + try: + gdf = geopandas.read_file(data, driver='GeoJSON') + copy.update({name: gdf}) + except Exception: + # if it fails, assume it is a csv string + # load csv string to a dataframe + df = pd.read_csv(data) + copy.update({name: df}) self.data = copy @@ -162,22 +244,27 @@ def show(self, data=None, config=None, read_only=False, center_map=False): map1.show() ''' - keplergl_html = resource_string(__name__, 'static/keplergl.html').decode('utf-8') + keplergl_html = resource_string( + __name__, 'static/keplergl.html').decode('utf-8') # find open of body k = keplergl_html.find("") - data_to_add = data_to_json(self.data, None) if data == None else data_to_json(data, None) - config_to_add = self.config if config == None else config + data_to_add = data_to_json(self.data if data is None else data, None) + + config_to_add = self.config if config is None else config - keplergl_data = json.dumps({"config": config_to_add, "data": data_to_add, "options": {"readOnly": read_only, "centerMap": center_map}}) + keplergl_data = json.dumps({"config": config_to_add, "data": data_to_add, "options": { + "readOnly": read_only, "centerMap": center_map}}) - cmd = """window.__keplerglDataConfig = {};""".format(keplergl_data) - frame_txt = keplergl_html[:k] + "" + keplergl_html[k+6:] + cmd = f"window.__keplerglDataConfig = {keplergl_data};" + frame_txt = keplergl_html[:k] + "" + keplergl_html[k+6:] if "google.colab" in sys.modules: - from IPython.display import HTML, Javascript + from IPython.display import HTML, Javascript display(HTML(frame_txt)) - display(Javascript(f"google.colab.output.setIframeHeight('{self.height}');")) + display(Javascript( + f"google.colab.output.setIframeHeight('{self.height}');")) def _repr_html_(self, data=None, config=None, read_only=False, center_map=False): ''' Return current map in an html encoded string @@ -199,20 +286,23 @@ def _repr_html_(self, data=None, config=None, read_only=False, center_map=False) keplergl._repr_html_() ''' - keplergl_html = resource_string(__name__, 'static/keplergl.html').decode('utf-8') + keplergl_html = resource_string( + __name__, 'static/keplergl.html').decode('utf-8') # find open of body k = keplergl_html.find("") - data_to_add = data_to_json(self.data, None) if data == None else data_to_json(data, None) - config_to_add = self.config if config == None else config + data_to_add = data_to_json(self.data if data is None else data, None) + config_to_add = self.config if config is None else config # for key in data_to_add: # print(type(data_to_add[key])) - keplergl_data = json.dumps({"config": config_to_add, "data": data_to_add, "options": {"readOnly": read_only, "centerMap": center_map}}) + keplergl_data = json.dumps({"config": config_to_add, "data": data_to_add, "options": { + "readOnly": read_only, "centerMap": center_map}}) - cmd = """window.__keplerglDataConfig = {};""".format(keplergl_data) - frame_txt = keplergl_html[:k] + "" + keplergl_html[k+6:] + cmd = f"window.__keplerglDataConfig = {keplergl_data};" + frame_txt = keplergl_html[:k] + "" + keplergl_html[k+6:] return frame_txt.encode('utf-8') @@ -236,9 +326,10 @@ def save_to_html(self, data=None, config=None, file_name='keplergl_map.html', re keplergl.save_to_html(file_name='first_map.html') ''' - frame_txt = self._repr_html_(data=data, config=config, read_only=read_only, center_map=center_map) + frame_txt = self._repr_html_( + data=data, config=config, read_only=read_only, center_map=center_map) with open(file_name, 'wb') as f: f.write(frame_txt) - print("Map saved to {}!".format(file_name)) + print(f"Map saved to {file_name}!") diff --git a/bindings/kepler.gl-jupyter/pyproject.toml b/bindings/kepler.gl-jupyter/pyproject.toml index 20789c6ba2..f549f1df67 100644 --- a/bindings/kepler.gl-jupyter/pyproject.toml +++ b/bindings/kepler.gl-jupyter/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["jupyter_packaging~=0.7.0", "jupyterlab>=3.0.0,==3.*", "setuptools>=40.8.0", "wheel"] -build-backend = "setuptools.build_meta" \ No newline at end of file +requires = ["geopandas~=0.14.3", "ipywidgets~=7.8.1", "notebook~=6.0.1", "jupyter_packaging~=0.12.3", "jupyterlab~=4.1.6", "pyarrow~=16.0.0", "setuptools>=69.5.1", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/bindings/kepler.gl-jupyter/requirements.txt b/bindings/kepler.gl-jupyter/requirements.txt index b78c442991..73022b14d4 100644 --- a/bindings/kepler.gl-jupyter/requirements.txt +++ b/bindings/kepler.gl-jupyter/requirements.txt @@ -1,6 +1,11 @@ -geopandas==0.5.0 -ipywidgets==7.4.2 -pandas==0.24.2 +geopandas==0.14.3 +ipywidgets==7.8.1 Shapely==1.6.4.post2 traitlets==4.3.2 traittypes==0.2.1 +jupyter_packaging==0.12.3 +notebook==6.0.1 +setuptools==69.5.1 +jupyter==1.0.0 +jupyterlab==4.1.6 +pyarrow==16.0.0 diff --git a/bindings/kepler.gl-jupyter/setup.py b/bindings/kepler.gl-jupyter/setup.py index 9e9628e56f..a3a50b0548 100644 --- a/bindings/kepler.gl-jupyter/setup.py +++ b/bindings/kepler.gl-jupyter/setup.py @@ -64,11 +64,18 @@ 'long_description': LONG_DESCRIPTION, 'include_package_data': True, 'install_requires': [ - 'ipywidgets>=7.0.0,<8', + 'ipywidgets>=7.8.1,<8', 'traittypes>=0.2.1', - 'geopandas>=0.5.0', - 'pandas>=0.23.0', - 'Shapely>=1.6.4.post2' + 'traitlets>=4.3.2', + 'geopandas>=0.14.3', + 'Shapely>=1.6.4.post2', + 'jupyter_packaging>=0.12.3', + 'jupyter>=1.0.0', + 'jupyterlab>=4.1.6', + 'notebook>=6.0.1', + 'pyarrow>=16.0.0', + 'geoarrow-pyarrow>=0.1.2', + 'geoarrow-pandas>=0.1.1', ], 'packages': find_packages(), 'zip_safe': False, diff --git a/website/webpack.config.js b/website/webpack.config.js index 2d18452399..bd20d99b11 100644 --- a/website/webpack.config.js +++ b/website/webpack.config.js @@ -75,11 +75,12 @@ const COMMON_CONFIG = { type: 'javascript/auto' }, // for compiling @probe.gl, website build started to fail (March, 2024) + // netlify biulder complains loader not found for these modules (April, 2024) { test: /\.(js)$/, loader: 'babel-loader', options: BABEL_CONFIG, - include: /node_modules\/@probe.gl/ + include: [/node_modules\/@probe.gl/, /node_modules\/@loaders.gl/, /node_modules\/@math.gl/] } ] },