Skip to content

Commit 1214aa8

Browse files
hanwengitster
hanwen
authored and
gitster
committed
reftable: add blocksource, an abstraction for random access reads
The reftable format is usually used with files for storage. However, we abstract away this using the blocksource data structure. This has two advantages: * log blocks are zlib compressed, and handling them is simplified if we can discard byte segments from within the block layer. * for unittests, it is useful to read and write in-memory. The blocksource allows us to abstract the data away from on-disk files. Signed-off-by: Han-Wen Nienhuys <hanwen@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
1 parent ef8a6c6 commit 1214aa8

File tree

4 files changed

+220
-0
lines changed

4 files changed

+220
-0
lines changed

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2451,6 +2451,7 @@ xdiff-objs: $(XDIFF_OBJS)
24512451

24522452
REFTABLE_OBJS += reftable/basics.o
24532453
REFTABLE_OBJS += reftable/error.o
2454+
REFTABLE_OBJS += reftable/blocksource.o
24542455
REFTABLE_OBJS += reftable/publicbasics.o
24552456

24562457
REFTABLE_TEST_OBJS += reftable/test_framework.o

‎reftable/blocksource.c‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/*
2+
Copyright 2020 Google LLC
3+
4+
Use of this source code is governed by a BSD-style
5+
license that can be found in the LICENSE file or at
6+
https://developers.google.com/open-source/licenses/bsd
7+
*/
8+
9+
#include "system.h"
10+
11+
#include "basics.h"
12+
#include "blocksource.h"
13+
#include "reftable-blocksource.h"
14+
#include "reftable-error.h"
15+
16+
static void strbuf_return_block(void *b, struct reftable_block *dest)
17+
{
18+
memset(dest->data, 0xff, dest->len);
19+
reftable_free(dest->data);
20+
}
21+
22+
static void strbuf_close(void *b)
23+
{
24+
}
25+
26+
static int strbuf_read_block(void *v, struct reftable_block *dest, uint64_t off,
27+
uint32_t size)
28+
{
29+
struct strbuf *b = v;
30+
assert(off + size <= b->len);
31+
dest->data = reftable_calloc(size);
32+
memcpy(dest->data, b->buf + off, size);
33+
dest->len = size;
34+
return size;
35+
}
36+
37+
static uint64_t strbuf_size(void *b)
38+
{
39+
return ((struct strbuf *)b)->len;
40+
}
41+
42+
static struct reftable_block_source_vtable strbuf_vtable = {
43+
.size = &strbuf_size,
44+
.read_block = &strbuf_read_block,
45+
.return_block = &strbuf_return_block,
46+
.close = &strbuf_close,
47+
};
48+
49+
void block_source_from_strbuf(struct reftable_block_source *bs,
50+
struct strbuf *buf)
51+
{
52+
assert(!bs->ops);
53+
bs->ops = &strbuf_vtable;
54+
bs->arg = buf;
55+
}
56+
57+
static void malloc_return_block(void *b, struct reftable_block *dest)
58+
{
59+
memset(dest->data, 0xff, dest->len);
60+
reftable_free(dest->data);
61+
}
62+
63+
static struct reftable_block_source_vtable malloc_vtable = {
64+
.return_block = &malloc_return_block,
65+
};
66+
67+
static struct reftable_block_source malloc_block_source_instance = {
68+
.ops = &malloc_vtable,
69+
};
70+
71+
struct reftable_block_source malloc_block_source(void)
72+
{
73+
return malloc_block_source_instance;
74+
}
75+
76+
struct file_block_source {
77+
int fd;
78+
uint64_t size;
79+
};
80+
81+
static uint64_t file_size(void *b)
82+
{
83+
return ((struct file_block_source *)b)->size;
84+
}
85+
86+
static void file_return_block(void *b, struct reftable_block *dest)
87+
{
88+
memset(dest->data, 0xff, dest->len);
89+
reftable_free(dest->data);
90+
}
91+
92+
static void file_close(void *b)
93+
{
94+
int fd = ((struct file_block_source *)b)->fd;
95+
if (fd > 0) {
96+
close(fd);
97+
((struct file_block_source *)b)->fd = 0;
98+
}
99+
100+
reftable_free(b);
101+
}
102+
103+
static int file_read_block(void *v, struct reftable_block *dest, uint64_t off,
104+
uint32_t size)
105+
{
106+
struct file_block_source *b = v;
107+
assert(off + size <= b->size);
108+
dest->data = reftable_malloc(size);
109+
if (pread(b->fd, dest->data, size, off) != size)
110+
return -1;
111+
dest->len = size;
112+
return size;
113+
}
114+
115+
static struct reftable_block_source_vtable file_vtable = {
116+
.size = &file_size,
117+
.read_block = &file_read_block,
118+
.return_block = &file_return_block,
119+
.close = &file_close,
120+
};
121+
122+
int reftable_block_source_from_file(struct reftable_block_source *bs,
123+
const char *name)
124+
{
125+
struct stat st = { 0 };
126+
int err = 0;
127+
int fd = open(name, O_RDONLY);
128+
struct file_block_source *p = NULL;
129+
if (fd < 0) {
130+
if (errno == ENOENT) {
131+
return REFTABLE_NOT_EXIST_ERROR;
132+
}
133+
return -1;
134+
}
135+
136+
err = fstat(fd, &st);
137+
if (err < 0)
138+
return -1;
139+
140+
p = reftable_calloc(sizeof(struct file_block_source));
141+
p->size = st.st_size;
142+
p->fd = fd;
143+
144+
assert(!bs->ops);
145+
bs->ops = &file_vtable;
146+
bs->arg = p;
147+
return 0;
148+
}

‎reftable/blocksource.h‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
Copyright 2020 Google LLC
3+
4+
Use of this source code is governed by a BSD-style
5+
license that can be found in the LICENSE file or at
6+
https://developers.google.com/open-source/licenses/bsd
7+
*/
8+
9+
#ifndef BLOCKSOURCE_H
10+
#define BLOCKSOURCE_H
11+
12+
#include "system.h"
13+
14+
struct reftable_block_source;
15+
16+
/* Create an in-memory block source for reading reftables */
17+
void block_source_from_strbuf(struct reftable_block_source *bs,
18+
struct strbuf *buf);
19+
20+
struct reftable_block_source malloc_block_source(void);
21+
22+
#endif

‎reftable/reftable-blocksource.h‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
Copyright 2020 Google LLC
3+
4+
Use of this source code is governed by a BSD-style
5+
license that can be found in the LICENSE file or at
6+
https://developers.google.com/open-source/licenses/bsd
7+
*/
8+
9+
#ifndef REFTABLE_BLOCKSOURCE_H
10+
#define REFTABLE_BLOCKSOURCE_H
11+
12+
#include <stdint.h>
13+
14+
/* block_source is a generic wrapper for a seekable readable file.
15+
*/
16+
struct reftable_block_source {
17+
struct reftable_block_source_vtable *ops;
18+
void *arg;
19+
};
20+
21+
/* a contiguous segment of bytes. It keeps track of its generating block_source
22+
* so it can return itself into the pool. */
23+
struct reftable_block {
24+
uint8_t *data;
25+
int len;
26+
struct reftable_block_source source;
27+
};
28+
29+
/* block_source_vtable are the operations that make up block_source */
30+
struct reftable_block_source_vtable {
31+
/* returns the size of a block source */
32+
uint64_t (*size)(void *source);
33+
34+
/* reads a segment from the block source. It is an error to read
35+
beyond the end of the block */
36+
int (*read_block)(void *source, struct reftable_block *dest,
37+
uint64_t off, uint32_t size);
38+
/* mark the block as read; may return the data back to malloc */
39+
void (*return_block)(void *source, struct reftable_block *blockp);
40+
41+
/* release all resources associated with the block source */
42+
void (*close)(void *source);
43+
};
44+
45+
/* opens a file on the file system as a block_source */
46+
int reftable_block_source_from_file(struct reftable_block_source *block_src,
47+
const char *name);
48+
49+
#endif

0 commit comments

Comments
 (0)