Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convenient function call for getting op name to stats #597

Merged
merged 4 commits into from
Feb 28, 2025
Merged

Convenient function call for getting op name to stats #597

Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2dd01e7
for executor function call
BeachWang Feb 27, 2025
658892c
for op to stat
BeachWang Feb 28, 2025
8bbc627
Merge branch 'main' of github.com:alibaba/data-juicer into refine/con…
BeachWang Feb 28, 2025
da1789a
use dataset.take
BeachWang Feb 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions data_juicer/utils/constant.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,23 @@ def __getattr__(cls, attr):
cls._accessed_by[caller_class].add(stat_key)
return stat_key

def get_access_log(cls, dj_cfg=None):
def get_access_log(cls, dj_cfg=None, dataset=None):
if cls._accessed_by:
return cls._accessed_by
elif dj_cfg and dataset:
tmp_dj_cfg = copy.deepcopy(dj_cfg)
tmp_dj_cfg.use_cache = False
tmp_dj_cfg.use_checkpoint = False

from data_juicer.core import Analyzer
tmp_analyzer = Analyzer(tmp_dj_cfg)

from data_juicer.core.data import NestedDataset
dataset = NestedDataset.from_dict(
{k: [v]
for k, v in dataset[0].items()})
# do not overwrite the true analysis results
tmp_analyzer.run(dataset=dataset, skip_export=True)
elif dj_cfg:
tmp_dj_cfg = copy.deepcopy(dj_cfg)
# the access has been skipped due to the use of cache
Expand Down Expand Up @@ -175,9 +189,6 @@ def get_access_log(cls, dj_cfg=None):
tmp_dj_cfg.use_cache = False
tmp_dj_cfg.use_checkpoint = False

from data_juicer.config import get_init_configs
tmp_dj_cfg = get_init_configs(tmp_dj_cfg)

from data_juicer.core import Analyzer
tmp_analyzer = Analyzer(tmp_dj_cfg)
# do not overwrite the true analysis results
Expand Down