Skip to content

Commit 56bdd78

Browse files
decrease parquet entity limit before chunking (#346)
1 parent a612c25 commit 56bdd78

File tree

1 file changed

+1
-1
lines changed

1 file changed

+1
-1
lines changed

digital_land/package/dataset_parquet.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -386,7 +386,7 @@ def load_entities(self, transformed_parquet_dir, resource_path, organisation_pat
386386
max_sql = f"select MAX(entity) FROM parquet_scan('{transformed_parquet_dir}/*.parquet');"
387387
max_entity = self.conn.execute(max_sql).fetchone()[0]
388388
total_entities = max_entity - min_entity
389-
entity_limit = 1000000
389+
entity_limit = 100000
390390
if total_entities > entity_limit:
391391
# create a temparary output path to store separate entity file in
392392
temp_dir = (

0 commit comments

Comments
 (0)