Skip to content

Commit 7c82f40

Browse files
committed
Merge branch 'master' into dev-branch
2 parents c4a8678 + 6f03a8c commit 7c82f40

File tree

7 files changed

+191
-58
lines changed

7 files changed

+191
-58
lines changed

tools/build.py

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@
6464

6565
# index.json is:
6666
# {
67-
# "v": 1, <-- file format version
67+
# "v": 2, <-- file format version
6868
# "updated": <utc-seconds-since-1970>,
6969
# "packages": {
7070
# {
@@ -78,7 +78,9 @@
7878
# "7": ["0.2", "0.3", "0.4"],
7979
# ... <-- Other bytecode versions
8080
# "py": ["0.1", "0.2", "0.3", "0.4"]
81-
# }
81+
# },
82+
# // The following entries were added in file format version 2.
83+
# path: "micropython/bluetooth/aioble",
8284
# },
8385
# ...
8486
# }
@@ -122,7 +124,7 @@
122124
import time
123125

124126

125-
_JSON_VERSION_INDEX = 1
127+
_JSON_VERSION_INDEX = 2
126128
_JSON_VERSION_PACKAGE = 1
127129

128130

@@ -268,7 +270,7 @@ def _copy_as_py(
268270

269271
# Update to the latest metadata, and add any new versions to the package in
270272
# the index json.
271-
def _update_index_package_metadata(index_package_json, metadata, mpy_version):
273+
def _update_index_package_metadata(index_package_json, metadata, mpy_version, package_path):
272274
index_package_json["version"] = metadata.version or ""
273275
index_package_json["author"] = "" # TODO: Make manifestfile.py capture this.
274276
index_package_json["description"] = metadata.description or ""
@@ -283,6 +285,9 @@ def _update_index_package_metadata(index_package_json, metadata, mpy_version):
283285
print(" New version {}={}".format(v, metadata.version))
284286
index_package_json["versions"][v].append(metadata.version)
285287

288+
# The following entries were added in file format version 2.
289+
index_package_json["path"] = package_path
290+
286291

287292
def build(output_path, hash_prefix_len, mpy_cross_path):
288293
import manifestfile
@@ -318,7 +323,8 @@ def build(output_path, hash_prefix_len, mpy_cross_path):
318323

319324
for lib_dir in lib_dirs:
320325
for manifest_path in glob.glob(os.path.join(lib_dir, "**", "manifest.py"), recursive=True):
321-
print("{}".format(os.path.dirname(manifest_path)))
326+
package_path = os.path.dirname(manifest_path)
327+
print("{}".format(package_path))
322328
# .../foo/manifest.py -> foo
323329
package_name = os.path.basename(os.path.dirname(manifest_path))
324330

@@ -342,7 +348,9 @@ def build(output_path, hash_prefix_len, mpy_cross_path):
342348
}
343349
index_json["packages"].append(index_package_json)
344350

345-
_update_index_package_metadata(index_package_json, manifest.metadata(), mpy_version)
351+
_update_index_package_metadata(
352+
index_package_json, manifest.metadata(), mpy_version, package_path
353+
)
346354

347355
# This is the package json that mip/mpremote downloads.
348356
mpy_package_json = {

tools/ci.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ function ci_package_tests_run {
6666
unix-ffi/gettext/test_gettext.py \
6767
unix-ffi/pwd/test_getpwnam.py \
6868
unix-ffi/re/test_re.py \
69+
unix-ffi/sqlite3/test_sqlite3.py \
70+
unix-ffi/sqlite3/test_sqlite3_2.py \
71+
unix-ffi/sqlite3/test_sqlite3_3.py \
6972
unix-ffi/time/test_strftime.py \
7073
; do
7174
echo "Running test $test"

unix-ffi/sqlite3/manifest.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
metadata(version="0.2.4")
1+
metadata(version="0.3.0")
22

33
# Originally written by Paul Sokolovsky.
44

unix-ffi/sqlite3/sqlite3.py

Lines changed: 125 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,29 @@
11
import sys
22
import ffilib
3+
import uctypes
34

45

56
sq3 = ffilib.open("libsqlite3")
67

8+
# int sqlite3_open(
9+
# const char *filename, /* Database filename (UTF-8) */
10+
# sqlite3 **ppDb /* OUT: SQLite db handle */
11+
# );
712
sqlite3_open = sq3.func("i", "sqlite3_open", "sp")
8-
# int sqlite3_close(sqlite3*);
9-
sqlite3_close = sq3.func("i", "sqlite3_close", "p")
13+
# int sqlite3_config(int, ...);
14+
sqlite3_config = sq3.func("i", "sqlite3_config", "ii")
15+
# int sqlite3_get_autocommit(sqlite3*);
16+
sqlite3_get_autocommit = sq3.func("i", "sqlite3_get_autocommit", "p")
17+
# int sqlite3_close_v2(sqlite3*);
18+
sqlite3_close = sq3.func("i", "sqlite3_close_v2", "p")
1019
# int sqlite3_prepare(
1120
# sqlite3 *db, /* Database handle */
1221
# const char *zSql, /* SQL statement, UTF-8 encoded */
1322
# int nByte, /* Maximum length of zSql in bytes. */
1423
# sqlite3_stmt **ppStmt, /* OUT: Statement handle */
1524
# const char **pzTail /* OUT: Pointer to unused portion of zSql */
1625
# );
17-
sqlite3_prepare = sq3.func("i", "sqlite3_prepare", "psipp")
26+
sqlite3_prepare = sq3.func("i", "sqlite3_prepare_v2", "psipp")
1827
# int sqlite3_finalize(sqlite3_stmt *pStmt);
1928
sqlite3_finalize = sq3.func("i", "sqlite3_finalize", "p")
2029
# int sqlite3_step(sqlite3_stmt*);
@@ -23,20 +32,17 @@
2332
sqlite3_column_count = sq3.func("i", "sqlite3_column_count", "p")
2433
# int sqlite3_column_type(sqlite3_stmt*, int iCol);
2534
sqlite3_column_type = sq3.func("i", "sqlite3_column_type", "pi")
35+
# int sqlite3_column_int(sqlite3_stmt*, int iCol);
2636
sqlite3_column_int = sq3.func("i", "sqlite3_column_int", "pi")
27-
# using "d" return type gives wrong results
37+
# double sqlite3_column_double(sqlite3_stmt*, int iCol);
2838
sqlite3_column_double = sq3.func("d", "sqlite3_column_double", "pi")
39+
# const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol);
2940
sqlite3_column_text = sq3.func("s", "sqlite3_column_text", "pi")
3041
# sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*);
31-
# TODO: should return long int
32-
sqlite3_last_insert_rowid = sq3.func("i", "sqlite3_last_insert_rowid", "p")
42+
sqlite3_last_insert_rowid = sq3.func("l", "sqlite3_last_insert_rowid", "p")
3343
# const char *sqlite3_errmsg(sqlite3*);
3444
sqlite3_errmsg = sq3.func("s", "sqlite3_errmsg", "p")
3545

36-
# Too recent
37-
##const char *sqlite3_errstr(int);
38-
# sqlite3_errstr = sq3.func("s", "sqlite3_errstr", "i")
39-
4046

4147
SQLITE_OK = 0
4248
SQLITE_ERROR = 1
@@ -51,6 +57,11 @@
5157
SQLITE_BLOB = 4
5258
SQLITE_NULL = 5
5359

60+
SQLITE_CONFIG_URI = 17
61+
62+
# For compatibility with CPython sqlite3 driver
63+
LEGACY_TRANSACTION_CONTROL = -1
64+
5465

5566
class Error(Exception):
5667
pass
@@ -61,79 +72,142 @@ def check_error(db, s):
6172
raise Error(s, sqlite3_errmsg(db))
6273

6374

75+
def get_ptr_size():
76+
return uctypes.sizeof({"ptr": (0 | uctypes.PTR, uctypes.PTR)})
77+
78+
79+
def __prepare_stmt(db, sql):
80+
# Prepares a statement
81+
stmt_ptr = bytes(get_ptr_size())
82+
res = sqlite3_prepare(db, sql, -1, stmt_ptr, None)
83+
check_error(db, res)
84+
return int.from_bytes(stmt_ptr, sys.byteorder)
85+
86+
def __exec_stmt(db, sql):
87+
# Prepares, executes, and finalizes a statement
88+
stmt = __prepare_stmt(db, sql)
89+
sqlite3_step(stmt)
90+
res = sqlite3_finalize(stmt)
91+
check_error(db, res)
92+
93+
def __is_dml(sql):
94+
# Checks if a sql query is a DML, as these get a BEGIN in LEGACY_TRANSACTION_CONTROL
95+
for dml in ["INSERT", "DELETE", "UPDATE", "MERGE"]:
96+
if dml in sql.upper():
97+
return True
98+
return False
99+
100+
64101
class Connections:
65-
def __init__(self, h):
66-
self.h = h
102+
def __init__(self, db, isolation_level, autocommit):
103+
self.db = db
104+
self.isolation_level = isolation_level
105+
self.autocommit = autocommit
106+
107+
def commit(self):
108+
if self.autocommit == LEGACY_TRANSACTION_CONTROL and not sqlite3_get_autocommit(self.db):
109+
__exec_stmt(self.db, "COMMIT")
110+
elif self.autocommit == False:
111+
__exec_stmt(self.db, "COMMIT")
112+
__exec_stmt(self.db, "BEGIN")
113+
114+
def rollback(self):
115+
if self.autocommit == LEGACY_TRANSACTION_CONTROL and not sqlite3_get_autocommit(self.db):
116+
__exec_stmt(self.db, "ROLLBACK")
117+
elif self.autocommit == False:
118+
__exec_stmt(self.db, "ROLLBACK")
119+
__exec_stmt(self.db, "BEGIN")
67120

68121
def cursor(self):
69-
return Cursor(self.h)
122+
return Cursor(self.db, self.isolation_level, self.autocommit)
70123

71124
def close(self):
72-
s = sqlite3_close(self.h)
73-
check_error(self.h, s)
125+
if self.db:
126+
if self.autocommit == False and not sqlite3_get_autocommit(self.db):
127+
__exec_stmt(self.db, "ROLLBACK")
128+
129+
res = sqlite3_close(self.db)
130+
check_error(self.db, res)
131+
self.db = None
74132

75133

76134
class Cursor:
77-
def __init__(self, h):
78-
self.h = h
79-
self.stmnt = None
135+
def __init__(self, db, isolation_level, autocommit):
136+
self.db = db
137+
self.isolation_level = isolation_level
138+
self.autocommit = autocommit
139+
self.stmt = None
140+
141+
def __quote(val):
142+
if isinstance(val, str):
143+
return "'%s'" % val
144+
return str(val)
80145

81146
def execute(self, sql, params=None):
147+
if self.stmt:
148+
# If there is an existing statement, finalize that to free it
149+
res = sqlite3_finalize(self.stmt)
150+
check_error(self.db, res)
151+
82152
if params:
83-
params = [quote(v) for v in params]
153+
params = [self.__quote(v) for v in params]
84154
sql = sql % tuple(params)
85-
print(sql)
86-
b = bytearray(4)
87-
s = sqlite3_prepare(self.h, sql, -1, b, None)
88-
check_error(self.h, s)
89-
self.stmnt = int.from_bytes(b, sys.byteorder)
90-
# print("stmnt", self.stmnt)
91-
self.num_cols = sqlite3_column_count(self.stmnt)
92-
# print("num_cols", self.num_cols)
93-
# If it's not select, actually execute it here
94-
# num_cols == 0 for statements which don't return data (=> modify it)
155+
156+
if __is_dml(sql) and self.autocommit == LEGACY_TRANSACTION_CONTROL and sqlite3_get_autocommit(self.db):
157+
# For compatibility with CPython, add functionality for their default transaction
158+
# behavior. Changing autocommit from LEGACY_TRANSACTION_CONTROL will remove this
159+
__exec_stmt(self.db, "BEGIN " + self.isolation_level)
160+
161+
self.stmt = __prepare_stmt(self.db, sql)
162+
self.num_cols = sqlite3_column_count(self.stmt)
163+
95164
if not self.num_cols:
96165
v = self.fetchone()
166+
# If it's not select, actually execute it here
167+
# num_cols == 0 for statements which don't return data (=> modify it)
97168
assert v is None
98-
self.lastrowid = sqlite3_last_insert_rowid(self.h)
169+
self.lastrowid = sqlite3_last_insert_rowid(self.db)
99170

100171
def close(self):
101-
s = sqlite3_finalize(self.stmnt)
102-
check_error(self.h, s)
172+
if self.stmt:
173+
res = sqlite3_finalize(self.stmt)
174+
check_error(self.db, res)
175+
self.stmt = None
103176

104-
def make_row(self):
177+
def __make_row(self):
105178
res = []
106179
for i in range(self.num_cols):
107-
t = sqlite3_column_type(self.stmnt, i)
108-
# print("type", t)
180+
t = sqlite3_column_type(self.stmt, i)
109181
if t == SQLITE_INTEGER:
110-
res.append(sqlite3_column_int(self.stmnt, i))
182+
res.append(sqlite3_column_int(self.stmt, i))
111183
elif t == SQLITE_FLOAT:
112-
res.append(sqlite3_column_double(self.stmnt, i))
184+
res.append(sqlite3_column_double(self.stmt, i))
113185
elif t == SQLITE_TEXT:
114-
res.append(sqlite3_column_text(self.stmnt, i))
186+
res.append(sqlite3_column_text(self.stmt, i))
115187
else:
116188
raise NotImplementedError
117189
return tuple(res)
118190

119191
def fetchone(self):
120-
res = sqlite3_step(self.stmnt)
121-
# print("step:", res)
192+
res = sqlite3_step(self.stmt)
122193
if res == SQLITE_DONE:
123194
return None
124195
if res == SQLITE_ROW:
125-
return self.make_row()
126-
check_error(self.h, res)
196+
return self.__make_row()
197+
check_error(self.db, res)
198+
199+
200+
def connect(fname, uri=False, isolation_level="", autocommit=LEGACY_TRANSACTION_CONTROL):
201+
if isolation_level not in [None, "", "DEFERRED", "IMMEDIATE", "EXCLUSIVE"]:
202+
raise Error("Invalid option for isolation level")
127203

204+
sqlite3_config(SQLITE_CONFIG_URI, int(uri))
128205

129-
def connect(fname):
130-
b = bytearray(4)
131-
sqlite3_open(fname, b)
132-
h = int.from_bytes(b, sys.byteorder)
133-
return Connections(h)
206+
sqlite_ptr = bytes(get_ptr_size())
207+
sqlite3_open(fname, sqlite_ptr)
208+
db = int.from_bytes(sqlite_ptr, sys.byteorder)
134209

210+
if autocommit == False:
211+
__exec_stmt(db, "BEGIN")
135212

136-
def quote(val):
137-
if isinstance(val, str):
138-
return "'%s'" % val
139-
return str(val)
213+
return Connections(db, isolation_level, autocommit)

unix-ffi/sqlite3/test_sqlite3.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,6 @@
1717
assert row == e
1818

1919
assert expected == []
20+
21+
cur.close()
22+
conn.close()

unix-ffi/sqlite3/test_sqlite3_2.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,6 @@
1010
cur.execute("SELECT * FROM foo")
1111
assert cur.fetchone() == (42,)
1212
assert cur.fetchone() is None
13+
14+
cur.close()
15+
conn.close()

unix-ffi/sqlite3/test_sqlite3_3.py

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import sqlite3
2+
3+
4+
def test_autocommit():
5+
conn = sqlite3.connect(":memory:", autocommit=True)
6+
7+
# First cursor creates table and inserts value (DML)
8+
cur = conn.cursor()
9+
cur.execute("CREATE TABLE foo(a int)")
10+
cur.execute("INSERT INTO foo VALUES (42)")
11+
cur.close()
12+
13+
# Second cursor fetches 42 due to the autocommit
14+
cur = conn.cursor()
15+
cur.execute("SELECT * FROM foo")
16+
assert cur.fetchone() == (42,)
17+
assert cur.fetchone() is None
18+
19+
cur.close()
20+
conn.close()
21+
22+
def test_manual():
23+
conn = sqlite3.connect(":memory:", autocommit=False)
24+
25+
# First cursor creates table, insert rolls back
26+
cur = conn.cursor()
27+
cur.execute("CREATE TABLE foo(a int)")
28+
conn.commit()
29+
cur.execute("INSERT INTO foo VALUES (42)")
30+
cur.close()
31+
conn.rollback()
32+
33+
# Second connection fetches nothing due to the rollback
34+
cur = conn.cursor()
35+
cur.execute("SELECT * FROM foo")
36+
assert cur.fetchone() is None
37+
38+
cur.close()
39+
conn.close()
40+
41+
test_autocommit()
42+
test_manual()

0 commit comments

Comments
 (0)