Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

updated Meyer 2024 sparql comment #12

Merged
merged 1 commit into from
Nov 22, 2024
Merged

updated Meyer 2024 sparql comment #12

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions aksw.bib
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12563,12 +12563,12 @@ @InProceedings{Kilic2024TowardsRegionalPublic
url = {https://ceur-ws.org/Vol-3707/D2R224_paper_7.pdf},
}

@InCollection{Meyer2024AssessingSparqlCapabilititesLLM,
@InProceedings{Meyer2024AssessingSparqlCapabilititesLLM,
author = {Lars-Peter Meyer and Johannes Frey and Felix Brei and Natanael Arndt},
title = {Assessing SPARQL capabilities of Large Language Models},
year = {2024},
abstract = {The integration of Large Language Models (LLMs) with Knowledge Graphs (KGs) offers significant synergistic potential for knowledge-driven applications. One possible integration is the interpretation and generation of formal languages, such as those used in the Semantic Web, with SPARQL being a core technology for accessing KGs. In this paper, we focus on measuring out-of-the box capabilities of LLMs to work with SPARQL and more specifically with SPARQL SELECT queries applying a quantitative approach. We implemented various benchmarking tasks in the LLM-KG-Bench framework for automated execution and evaluation with several LLMs. The tasks assess capabilities along the dimensions of syntax, semantic read, semantic create, and the role of knowledge graph prompt inclusion. With this new benchmarking tasks, we evaluated a selection of GPT, Gemini, and Claude models. Our findings indicate that working with SPARQL SELECT queries is still challenging for LLMs and heavily depends on the specific LLM as well as the complexity of the task. While fixing basic syntax errors seems to pose no problems for the best of the current LLMs evaluated, creating semantically correct SPARQL SELECT queries is difficult in several cases.},
comment = {to appear in Proceedings of Workshop NLP4KGC @SEMANTICS 2024 at https://ceur-ws.org},
comment = {to appear in Proceedings of Workshop NLP4KGC, colocated with SEMANTICS 2024, at https://ceur-ws.org},
doi = {10.48550/ARXIV.2409.05925},
keywords = {group_aksw sys:relevantFor:infai es lpmeyer brei frey arndt},
url = {https://arxiv.org/pdf/2409.05925},
Expand Down