Skip to content

Commit e58cf84

Browse files
youknowoneyouknowone
authored
Update dbm from 3.13.11, implement sqlite3 autocommit (#6616)
1 parent a4df238 commit e58cf84

File tree

9 files changed

+851
-95
lines changed

9 files changed

+851
-95
lines changed

‎Lib/dbm/__init__.py‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import dbm
66
d = dbm.open(file, 'w', 0o666)
77
8-
The returned object is a dbm.gnu, dbm.ndbm or dbm.dumb object, dependent on the
8+
The returned object is a dbm.sqlite3, dbm.gnu, dbm.ndbm or dbm.dumb database object, dependent on the
99
type of database being opened (determined by the whichdb function) in the case
1010
of an existing dbm. If the dbm does not exist and the create or new flag ('c'
1111
or 'n') was specified, the dbm type will be determined by the availability of
@@ -38,7 +38,7 @@
3838
class error(Exception):
3939
pass
4040

41-
_names = ['dbm.gnu', 'dbm.ndbm', 'dbm.dumb']
41+
_names = ['dbm.sqlite3', 'dbm.gnu', 'dbm.ndbm', 'dbm.dumb']
4242
_defaultmod = None
4343
_modules = {}
4444

@@ -109,17 +109,18 @@ def whichdb(filename):
109109
"""
110110

111111
# Check for ndbm first -- this has a .pag and a .dir file
112+
filename = os.fsencode(filename)
112113
try:
113-
f = io.open(filename + ".pag", "rb")
114+
f = io.open(filename + b".pag", "rb")
114115
f.close()
115-
f = io.open(filename + ".dir", "rb")
116+
f = io.open(filename + b".dir", "rb")
116117
f.close()
117118
return "dbm.ndbm"
118119
except OSError:
119120
# some dbm emulations based on Berkeley DB generate a .db file
120121
# some do not, but they should be caught by the bsd checks
121122
try:
122-
f = io.open(filename + ".db", "rb")
123+
f = io.open(filename + b".db", "rb")
123124
f.close()
124125
# guarantee we can actually open the file using dbm
125126
# kind of overkill, but since we are dealing with emulations
@@ -134,12 +135,12 @@ def whichdb(filename):
134135
# Check for dumbdbm next -- this has a .dir and a .dat file
135136
try:
136137
# First check for presence of files
137-
os.stat(filename + ".dat")
138-
size = os.stat(filename + ".dir").st_size
138+
os.stat(filename + b".dat")
139+
size = os.stat(filename + b".dir").st_size
139140
# dumbdbm files with no keys are empty
140141
if size == 0:
141142
return "dbm.dumb"
142-
f = io.open(filename + ".dir", "rb")
143+
f = io.open(filename + b".dir", "rb")
143144
try:
144145
if f.read(1) in (b"'", b'"'):
145146
return "dbm.dumb"
@@ -163,6 +164,10 @@ def whichdb(filename):
163164
if len(s) != 4:
164165
return ""
165166

167+
# Check for SQLite3 header string.
168+
if s16 == b"SQLite format 3\0":
169+
return "dbm.sqlite3"
170+
166171
# Convert to 4-byte int in native byte order -- return "" if impossible
167172
try:
168173
(magic,) = struct.unpack("=l", s)

‎Lib/dbm/dumb.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ class _Database(collections.abc.MutableMapping):
4646
_io = _io # for _commit()
4747

4848
def __init__(self, filebasename, mode, flag='c'):
49+
filebasename = self._os.fsencode(filebasename)
4950
self._mode = mode
5051
self._readonly = (flag == 'r')
5152

@@ -54,14 +55,14 @@ def __init__(self, filebasename, mode, flag='c'):
5455
# where key is the string key, pos is the offset into the dat
5556
# file of the associated value's first byte, and siz is the number
5657
# of bytes in the associated value.
57-
self._dirfile = filebasename + '.dir'
58+
self._dirfile = filebasename + b'.dir'
5859

5960
# The data file is a binary file pointed into by the directory
6061
# file, and holds the values associated with keys. Each value
6162
# begins at a _BLOCKSIZE-aligned byte offset, and is a raw
6263
# binary 8-bit string value.
63-
self._datfile = filebasename + '.dat'
64-
self._bakfile = filebasename + '.bak'
64+
self._datfile = filebasename + b'.dat'
65+
self._bakfile = filebasename + b'.bak'
6566

6667
# The index is an in-memory dict, mirroring the directory file.
6768
self._index = None # maps keys to (pos, siz) pairs
@@ -97,7 +98,8 @@ def _update(self, flag):
9798
except OSError:
9899
if flag not in ('c', 'n'):
99100
raise
100-
self._modified = True
101+
with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f:
102+
self._chmod(self._dirfile)
101103
else:
102104
with f:
103105
for line in f:
@@ -133,6 +135,7 @@ def _commit(self):
133135
# position; UTF-8, though, does care sometimes.
134136
entry = "%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair)
135137
f.write(entry)
138+
self._modified = False
136139

137140
sync = _commit
138141

‎Lib/dbm/gnu.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
"""Provide the _gdbm module as a dbm submodule."""
2+
3+
from _gdbm import *

‎Lib/dbm/ndbm.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
"""Provide the _dbm module as a dbm submodule."""
2+
3+
from _dbm import *

‎Lib/dbm/sqlite3.py‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
import os
2+
import sqlite3
3+
from pathlib import Path
4+
from contextlib import suppress, closing
5+
from collections.abc import MutableMapping
6+
7+
BUILD_TABLE = """
8+
CREATE TABLE IF NOT EXISTS Dict (
9+
key BLOB UNIQUE NOT NULL,
10+
value BLOB NOT NULL
11+
)
12+
"""
13+
GET_SIZE = "SELECT COUNT (key) FROM Dict"
14+
LOOKUP_KEY = "SELECT value FROM Dict WHERE key = CAST(? AS BLOB)"
15+
STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))"
16+
DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)"
17+
ITER_KEYS = "SELECT key FROM Dict"
18+
19+
20+
class error(OSError):
21+
pass
22+
23+
24+
_ERR_CLOSED = "DBM object has already been closed"
25+
_ERR_REINIT = "DBM object does not support reinitialization"
26+
27+
28+
def _normalize_uri(path):
29+
path = Path(path)
30+
uri = path.absolute().as_uri()
31+
while "//" in uri:
32+
uri = uri.replace("//", "/")
33+
return uri
34+
35+
36+
class _Database(MutableMapping):
37+
38+
def __init__(self, path, /, *, flag, mode):
39+
if hasattr(self, "_cx"):
40+
raise error(_ERR_REINIT)
41+
42+
path = os.fsdecode(path)
43+
match flag:
44+
case "r":
45+
flag = "ro"
46+
case "w":
47+
flag = "rw"
48+
case "c":
49+
flag = "rwc"
50+
Path(path).touch(mode=mode, exist_ok=True)
51+
case "n":
52+
flag = "rwc"
53+
Path(path).unlink(missing_ok=True)
54+
Path(path).touch(mode=mode)
55+
case _:
56+
raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n', "
57+
f"not {flag!r}")
58+
59+
# We use the URI format when opening the database.
60+
uri = _normalize_uri(path)
61+
uri = f"{uri}?mode={flag}"
62+
if flag == "ro":
63+
# Add immutable=1 to allow read-only SQLite access even if wal/shm missing
64+
uri += "&immutable=1"
65+
66+
try:
67+
self._cx = sqlite3.connect(uri, autocommit=True, uri=True)
68+
except sqlite3.Error as exc:
69+
raise error(str(exc))
70+
71+
if flag != "ro":
72+
# This is an optimization only; it's ok if it fails.
73+
with suppress(sqlite3.OperationalError):
74+
self._cx.execute("PRAGMA journal_mode = wal")
75+
76+
if flag == "rwc":
77+
self._execute(BUILD_TABLE)
78+
79+
def _execute(self, *args, **kwargs):
80+
if not self._cx:
81+
raise error(_ERR_CLOSED)
82+
try:
83+
return closing(self._cx.execute(*args, **kwargs))
84+
except sqlite3.Error as exc:
85+
raise error(str(exc))
86+
87+
def __len__(self):
88+
with self._execute(GET_SIZE) as cu:
89+
row = cu.fetchone()
90+
return row[0]
91+
92+
def __getitem__(self, key):
93+
with self._execute(LOOKUP_KEY, (key,)) as cu:
94+
row = cu.fetchone()
95+
if not row:
96+
raise KeyError(key)
97+
return row[0]
98+
99+
def __setitem__(self, key, value):
100+
self._execute(STORE_KV, (key, value))
101+
102+
def __delitem__(self, key):
103+
with self._execute(DELETE_KEY, (key,)) as cu:
104+
if not cu.rowcount:
105+
raise KeyError(key)
106+
107+
def __iter__(self):
108+
try:
109+
with self._execute(ITER_KEYS) as cu:
110+
for row in cu:
111+
yield row[0]
112+
except sqlite3.Error as exc:
113+
raise error(str(exc))
114+
115+
def close(self):
116+
if self._cx:
117+
self._cx.close()
118+
self._cx = None
119+
120+
def keys(self):
121+
return list(super().keys())
122+
123+
def __enter__(self):
124+
return self
125+
126+
def __exit__(self, *args):
127+
self.close()
128+
129+
130+
def open(filename, /, flag="r", mode=0o666):
131+
"""Open a dbm.sqlite3 database and return the dbm object.
132+
133+
The 'filename' parameter is the name of the database file.
134+
135+
The optional 'flag' parameter can be one of ...:
136+
'r' (default): open an existing database for read only access
137+
'w': open an existing database for read/write access
138+
'c': create a database if it does not exist; open for read/write access
139+
'n': always create a new, empty database; open for read/write access
140+
141+
The optional 'mode' parameter is the Unix file access mode of the database;
142+
only used when creating a new database. Default: 0o666.
143+
"""
144+
return _Database(filename, flag=flag, mode=mode)

0 commit comments

Comments
 (0)