generated from Cloud-CV/EvalAI-Starters
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathchallenge_config.yaml
executable file
·201 lines (197 loc) · 7.95 KB
/
challenge_config.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
# If you are not sure what all these fields mean, please refer our documentation here:
# https://evalai.readthedocs.io/en/latest/configuration.html
title: Object Instance Detection @ ACCV2024
short_description: Instance Detection
description: templates/description.html
evaluation_details: templates/evaluation_details.html
terms_and_conditions: templates/terms_and_conditions.html
image: logo.jpg
submission_guidelines: templates/submission_guidelines.html
leaderboard_description: View the Real-world instance detection leaderboard here. For all metrics, higher number(%) means better performance. We rank methods w.r.t the primary metric, i.e., AP.
evaluation_script: evaluation_script.zip
remote_evaluation: False
is_docker_based: False
start_date: 2024-09-10 00:00:00
end_date: 2099-10-25 23:59:59
published: True
leaderboard:
- id: 1
schema:
{
"labels": ["AP", "AP50", "AP75", "AP_easy", "AP_hard", "AP_small", "AP_medium", "AP_large", "AR_1", "AR_10", "AR_100", "AR_small", "AR_medium", "AR_large"],
"default_order_by": "AP",
"metadata": {
"AP": {
"sort_ascending": False,
"description": "AP averages the precision over all instances at IoU threshold from 0.5 to 0.95 with the step size 0.05.",
},
"AP50": {
"sort_ascending": False,
"description": "AP50 is the precision averaged over all instances with IoU threshold as 0.5.",
},
"AP75": {
"sort_ascending": False,
"description": "AP75 is the precision averaged over all instances with IoU threshold as 0.75.",
},
"AP_easy": {
"sort_ascending": False,
"description": "AP_easy is the AP on <i>easy</i> scene images.",
},
"AP_hard": {
"sort_ascending": False,
"description": "AP_easy is the AP on <i>hard</i> scene images.",
},
"AP_small": {
"sort_ascending": False,
"description": "AP_easy is the AP on <i>small</i> object instances.",
},
"AP_medium": {
"sort_ascending": False,
"description": "AP_medium is the AP on <i>medium</i> object instances.",
},
"AP_large": {
"sort_ascending": False,
"description": "AP_hard is the AP on <i>hard</i> object instances.",
},
"AR_1": {
"sort_ascending": False,
"description": "AR averages the proposal recall at IoU threshold from 0.5 to 1.0 with the step size 0.05, given 1 detection per image.",
},
"AR_10": {
"sort_ascending": False,
"description": "AR averages the proposal recall at IoU threshold from 0.5 to 1.0 with the step size 0.05, given 10 detection per image.",
},
"AR_100": {
"sort_ascending": False,
"description": "AR averages the proposal recall at IoU threshold from 0.5 to 1.0 with the step size 0.05, given 100 detection per image.",
},
"AR_small": {
"sort_ascending": False,
"description": "AR averages the proposal recall at IoU threshold from 0.5 to 1.0 with the step size 0.05, for small instances.",
},
"AR_medium": {
"sort_ascending": False,
"description": "AR averages the proposal recall at IoU threshold from 0.5 to 1.0 with the step size 0.05, for medium instances.",
},
"AR_large": {
"sort_ascending": False,
"description": "AR averages the proposal recall at IoU threshold from 0.5 to 1.0 with the step size 0.05, for large instances.",
},
}
}
challenge_phases:
- id: 1
name: Dev Phase
description: templates/challenge_phase_1_description.html
leaderboard_public: False
is_public: True
is_submission_public: True
start_date: 2024-09-10 00:00:00
end_date: 2099-10-25 23:59:59
test_annotation_file: annotations/test_annotations_devsplit.json
codename: dev
max_submissions_per_day: 5
max_submissions_per_month: 100
max_submissions: 100
default_submission_meta_attributes:
- name: method_name
is_visible: True
- name: method_description
is_visible: True
- name: project_url
is_visible: True
- name: publication_url
is_visible: True
submission_meta_attributes:
- name: Did you use any pertained models, e.g., vision-language models (CLIP), vision models (DINOv2), etc?
description: Yes or No? If Yes, what models?
type: text
required: True
- name: Did you use additional training data, e.g., any data collected by yourself?
description: Yes or No? If Yes, what data?
type: text
- name: What types of GPUs did you use (e.g., Nvidia 3080) and how many GPUs did you use?
description: None
type: text
required: True
- name: If you are invited to present your work at our workshop, do you agree to submit a report and open-source your code for our verification?
description: Yes or No?
type: boolean
required: True
- name: Any remarks?
description: Please let us know any other info you want to share!
type: text
required: False
is_restricted_to_select_one_submission: False
is_partial_submission_evaluation_enabled: False
allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz"
- id: 2
name: Test Phase
description: templates/challenge_phase_2_description.html
leaderboard_public: True
is_public: True
is_submission_public: True
start_date: 2024-09-10 00:00:00
end_date: 2099-11-25 23:59:59
test_annotation_file: annotations/test_annotations_testsplit.json
codename: test
max_submissions_per_day: 5
max_submissions_per_month: 50
max_submissions: 50
default_submission_meta_attributes:
- name: method_name
is_visible: True
- name: method_description
is_visible: True
- name: project_url
is_visible: True
- name: publication_url
is_visible: True
submission_meta_attributes:
- name: Did you use any pertained models, e.g., vision-language models (CLIP), vision models (DINOv2), etc?
description: Yes or No? If Yes, what models?
type: text
required: True
- name: Did you use additional training data, e.g., any data collected by yourself?
description: Yes or No? If Yes, what data?
type: text
- name: What types of GPUs did you use (e.g., Nvidia 3080) and how many GPUs did you use?
description: None
type: text
required: True
- name: If you are invited to present your work at our workshop, do you agree to submit a report and open-source your code for our verification?
description: Yes or No?
type: boolean
required: True
- name: Any remarks?
description: Please let us know any other info you want to share!
type: text
required: False
is_restricted_to_select_one_submission: False
is_partial_submission_evaluation_enabled: False
dataset_splits:
- id: 1
name: Dev Split
codename: val_split
- id: 2
name: Test Split
codename: test_split
challenge_phase_splits:
- challenge_phase_id: 1
leaderboard_id: 1
dataset_split_id: 1
visibility: 3
leaderboard_decimal_precision: 2
is_leaderboard_order_descending: True
- challenge_phase_id: 2
leaderboard_id: 1
dataset_split_id: 1
visibility: 1
leaderboard_decimal_precision: 2
is_leaderboard_order_descending: True
- challenge_phase_id: 2
leaderboard_id: 1
dataset_split_id: 2
visibility: 3
leaderboard_decimal_precision: 2
is_leaderboard_order_descending: True