##// END OF EJS Templates
cext: add support for revlogv2...
Raphaël Gomès -
r47442:358737ab default
parent child Browse files
Show More
@@ -638,7 +638,7 b' static char parsers_doc[] = "Efficient c'
638 638 PyObject *encodedir(PyObject *self, PyObject *args);
639 639 PyObject *pathencode(PyObject *self, PyObject *args);
640 640 PyObject *lowerencode(PyObject *self, PyObject *args);
641 PyObject *parse_index2(PyObject *self, PyObject *args);
641 PyObject *parse_index2(PyObject *self, PyObject *args, PyObject *kwargs);
642 642
643 643 static PyMethodDef methods[] = {
644 644 {"pack_dirstate", pack_dirstate, METH_VARARGS, "pack a dirstate\n"},
@@ -646,7 +646,8 b' static PyMethodDef methods[] = {'
646 646 "create a set containing non-normal and other parent entries of given "
647 647 "dirstate\n"},
648 648 {"parse_dirstate", parse_dirstate, METH_VARARGS, "parse a dirstate\n"},
649 {"parse_index2", parse_index2, METH_VARARGS, "parse a revlog index\n"},
649 {"parse_index2", (PyCFunction)parse_index2, METH_VARARGS | METH_KEYWORDS,
650 "parse a revlog index\n"},
650 651 {"isasciistr", isasciistr, METH_VARARGS, "check if an ASCII string\n"},
651 652 {"asciilower", asciilower, METH_VARARGS, "lowercase an ASCII string\n"},
652 653 {"asciiupper", asciiupper, METH_VARARGS, "uppercase an ASCII string\n"},
@@ -98,6 +98,7 b' struct indexObjectStruct {'
98 98 int ntlookups; /* # lookups */
99 99 int ntmisses; /* # lookups that miss the cache */
100 100 int inlined;
101 long hdrsize; /* size of index headers. Differs in v1 v.s. v2 format */
101 102 };
102 103
103 104 static Py_ssize_t index_length(const indexObject *self)
@@ -113,14 +114,21 b' static Py_ssize_t inline_scan(indexObjec'
113 114 static int index_find_node(indexObject *self, const char *node);
114 115
115 116 #if LONG_MAX == 0x7fffffffL
116 static const char *const tuple_format = PY23("Kiiiiiis#", "Kiiiiiiy#");
117 static const char *const v1_tuple_format = PY23("Kiiiiiis#", "Kiiiiiiy#");
118 static const char *const v2_tuple_format =
119 PY23("Kiiiiiis#Ki", "Kiiiiiiy#Ki");
117 120 #else
118 static const char *const tuple_format = PY23("kiiiiiis#", "kiiiiiiy#");
121 static const char *const v1_tuple_format = PY23("kiiiiiis#", "kiiiiiiy#");
122 static const char *const v2_tuple_format =
123 PY23("kiiiiiis#ki", "kiiiiiiy#ki");
119 124 #endif
120 125
121 126 /* A RevlogNG v1 index entry is 64 bytes long. */
122 127 static const long v1_hdrsize = 64;
123 128
129 /* A Revlogv2 index entry is 96 bytes long. */
130 static const long v2_hdrsize = 96;
131
124 132 static void raise_revlog_error(void)
125 133 {
126 134 PyObject *mod = NULL, *dict = NULL, *errclass = NULL;
@@ -157,7 +165,7 b' cleanup:'
157 165 static const char *index_deref(indexObject *self, Py_ssize_t pos)
158 166 {
159 167 if (pos >= self->length)
160 return self->added + (pos - self->length) * v1_hdrsize;
168 return self->added + (pos - self->length) * self->hdrsize;
161 169
162 170 if (self->inlined && pos > 0) {
163 171 if (self->offsets == NULL) {
@@ -174,7 +182,7 b' static const char *index_deref(indexObje'
174 182 return self->offsets[pos];
175 183 }
176 184
177 return (const char *)(self->buf.buf) + pos * v1_hdrsize;
185 return (const char *)(self->buf.buf) + pos * self->hdrsize;
178 186 }
179 187
180 188 /*
@@ -280,8 +288,9 b' static inline int index_get_length(index'
280 288 */
281 289 static PyObject *index_get(indexObject *self, Py_ssize_t pos)
282 290 {
283 uint64_t offset_flags;
284 int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2;
291 uint64_t offset_flags, sidedata_offset;
292 int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2,
293 sidedata_comp_len;
285 294 const char *c_node_id;
286 295 const char *data;
287 296 Py_ssize_t length = index_length(self);
@@ -320,9 +329,19 b' static PyObject *index_get(indexObject *'
320 329 parent_2 = getbe32(data + 28);
321 330 c_node_id = data + 32;
322 331
323 return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len,
324 base_rev, link_rev, parent_1, parent_2, c_node_id,
325 self->nodelen);
332 if (self->hdrsize == v1_hdrsize) {
333 return Py_BuildValue(v1_tuple_format, offset_flags, comp_len,
334 uncomp_len, base_rev, link_rev, parent_1,
335 parent_2, c_node_id, self->nodelen);
336 } else {
337 sidedata_offset = getbe64(data + 64);
338 sidedata_comp_len = getbe32(data + 72);
339
340 return Py_BuildValue(v2_tuple_format, offset_flags, comp_len,
341 uncomp_len, base_rev, link_rev, parent_1,
342 parent_2, c_node_id, self->nodelen,
343 sidedata_offset, sidedata_comp_len);
344 }
326 345 }
327 346
328 347 /*
@@ -373,18 +392,30 b' static int node_check(Py_ssize_t nodelen'
373 392
374 393 static PyObject *index_append(indexObject *self, PyObject *obj)
375 394 {
376 uint64_t offset_flags;
395 uint64_t offset_flags, sidedata_offset;
377 396 int rev, comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2;
378 Py_ssize_t c_node_id_len;
397 Py_ssize_t c_node_id_len, sidedata_comp_len;
379 398 const char *c_node_id;
380 399 char *data;
381 400
382 if (!PyArg_ParseTuple(obj, tuple_format, &offset_flags, &comp_len,
383 &uncomp_len, &base_rev, &link_rev, &parent_1,
384 &parent_2, &c_node_id, &c_node_id_len)) {
401 if (self->hdrsize == v1_hdrsize) {
402 if (!PyArg_ParseTuple(obj, v1_tuple_format, &offset_flags,
403 &comp_len, &uncomp_len, &base_rev,
404 &link_rev, &parent_1, &parent_2,
405 &c_node_id, &c_node_id_len)) {
385 406 PyErr_SetString(PyExc_TypeError, "8-tuple required");
386 407 return NULL;
387 408 }
409 } else {
410 if (!PyArg_ParseTuple(
411 obj, v2_tuple_format, &offset_flags, &comp_len,
412 &uncomp_len, &base_rev, &link_rev, &parent_1, &parent_2,
413 &c_node_id, &c_node_id_len, &sidedata_offset, &sidedata_comp_len)) {
414 PyErr_SetString(PyExc_TypeError, "10-tuple required");
415 return NULL;
416 }
417 }
418
388 419 if (c_node_id_len != self->nodelen) {
389 420 PyErr_SetString(PyExc_TypeError, "invalid node");
390 421 return NULL;
@@ -393,15 +424,15 b' static PyObject *index_append(indexObjec'
393 424 if (self->new_length == self->added_length) {
394 425 size_t new_added_length =
395 426 self->added_length ? self->added_length * 2 : 4096;
396 void *new_added =
397 PyMem_Realloc(self->added, new_added_length * v1_hdrsize);
427 void *new_added = PyMem_Realloc(self->added, new_added_length *
428 self->hdrsize);
398 429 if (!new_added)
399 430 return PyErr_NoMemory();
400 431 self->added = new_added;
401 432 self->added_length = new_added_length;
402 433 }
403 434 rev = self->length + self->new_length;
404 data = self->added + v1_hdrsize * self->new_length++;
435 data = self->added + self->hdrsize * self->new_length++;
405 436 putbe32(offset_flags >> 32, data);
406 437 putbe32(offset_flags & 0xffffffffU, data + 4);
407 438 putbe32(comp_len, data + 8);
@@ -411,7 +442,14 b' static PyObject *index_append(indexObjec'
411 442 putbe32(parent_1, data + 24);
412 443 putbe32(parent_2, data + 28);
413 444 memcpy(data + 32, c_node_id, c_node_id_len);
445 /* Padding since SHA-1 is only 20 bytes for now */
414 446 memset(data + 32 + c_node_id_len, 0, 32 - c_node_id_len);
447 if (self->hdrsize != v1_hdrsize) {
448 putbe64(sidedata_offset, data + 64);
449 putbe32(sidedata_comp_len, data + 72);
450 /* Padding for 96 bytes alignment */
451 memset(data + 76, 0, self->hdrsize - 76);
452 }
415 453
416 454 if (self->ntinitialized)
417 455 nt_insert(&self->nt, c_node_id, rev);
@@ -2563,14 +2601,17 b' static Py_ssize_t inline_scan(indexObjec'
2563 2601 const char *data = (const char *)self->buf.buf;
2564 2602 Py_ssize_t pos = 0;
2565 2603 Py_ssize_t end = self->buf.len;
2566 long incr = v1_hdrsize;
2604 long incr = self->hdrsize;
2567 2605 Py_ssize_t len = 0;
2568 2606
2569 while (pos + v1_hdrsize <= end && pos >= 0) {
2570 uint32_t comp_len;
2607 while (pos + self->hdrsize <= end && pos >= 0) {
2608 uint32_t comp_len, sidedata_comp_len = 0;
2571 2609 /* 3rd element of header is length of compressed inline data */
2572 2610 comp_len = getbe32(data + pos + 8);
2573 incr = v1_hdrsize + comp_len;
2611 if (self->hdrsize == v2_hdrsize) {
2612 sidedata_comp_len = getbe32(data + pos + 72);
2613 }
2614 incr = self->hdrsize + comp_len + sidedata_comp_len;
2574 2615 if (offsets)
2575 2616 offsets[len] = data + pos;
2576 2617 len++;
@@ -2586,11 +2627,13 b' static Py_ssize_t inline_scan(indexObjec'
2586 2627 return len;
2587 2628 }
2588 2629
2589 static int index_init(indexObject *self, PyObject *args)
2630 static int index_init(indexObject *self, PyObject *args, PyObject *kwargs)
2590 2631 {
2591 PyObject *data_obj, *inlined_obj;
2632 PyObject *data_obj, *inlined_obj, *revlogv2;
2592 2633 Py_ssize_t size;
2593 2634
2635 static char *kwlist[] = {"data", "inlined", "revlogv2", NULL};
2636
2594 2637 /* Initialize before argument-checking to avoid index_dealloc() crash.
2595 2638 */
2596 2639 self->added = NULL;
@@ -2606,7 +2649,9 b' static int index_init(indexObject *self,'
2606 2649 self->nodelen = 20;
2607 2650 self->nullentry = NULL;
2608 2651
2609 if (!PyArg_ParseTuple(args, "OO", &data_obj, &inlined_obj))
2652 revlogv2 = NULL;
2653 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|O", kwlist,
2654 &data_obj, &inlined_obj, &revlogv2))
2610 2655 return -1;
2611 2656 if (!PyObject_CheckBuffer(data_obj)) {
2612 2657 PyErr_SetString(PyExc_TypeError,
@@ -2618,8 +2663,22 b' static int index_init(indexObject *self,'
2618 2663 return -1;
2619 2664 }
2620 2665
2621 self->nullentry = Py_BuildValue(PY23("iiiiiiis#", "iiiiiiiy#"), 0, 0, 0,
2622 -1, -1, -1, -1, nullid, self->nodelen);
2666 if (revlogv2 && PyObject_IsTrue(revlogv2)) {
2667 self->hdrsize = v2_hdrsize;
2668 } else {
2669 self->hdrsize = v1_hdrsize;
2670 }
2671
2672 if (self->hdrsize == v1_hdrsize) {
2673 self->nullentry =
2674 Py_BuildValue(PY23("iiiiiiis#", "iiiiiiiy#"), 0, 0, 0, -1,
2675 -1, -1, -1, nullid, self->nodelen);
2676 } else {
2677 self->nullentry = Py_BuildValue(
2678 PY23("iiiiiiis#ii", "iiiiiiiy#ii"), 0, 0, 0, -1, -1, -1,
2679 -1, nullid, self->nodelen, 0, 0);
2680 }
2681
2623 2682 if (!self->nullentry)
2624 2683 return -1;
2625 2684 PyObject_GC_UnTrack(self->nullentry);
@@ -2641,11 +2700,11 b' static int index_init(indexObject *self,'
2641 2700 goto bail;
2642 2701 self->length = len;
2643 2702 } else {
2644 if (size % v1_hdrsize) {
2703 if (size % self->hdrsize) {
2645 2704 PyErr_SetString(PyExc_ValueError, "corrupt index file");
2646 2705 goto bail;
2647 2706 }
2648 self->length = size / v1_hdrsize;
2707 self->length = size / self->hdrsize;
2649 2708 }
2650 2709
2651 2710 return 0;
@@ -2797,16 +2856,16 b' PyTypeObject HgRevlogIndex_Type = {'
2797 2856 };
2798 2857
2799 2858 /*
2800 * returns a tuple of the form (index, index, cache) with elements as
2859 * returns a tuple of the form (index, cache) with elements as
2801 2860 * follows:
2802 2861 *
2803 * index: an index object that lazily parses RevlogNG records
2862 * index: an index object that lazily parses Revlog (v1 or v2) records
2804 2863 * cache: if data is inlined, a tuple (0, index_file_content), else None
2805 2864 * index_file_content could be a string, or a buffer
2806 2865 *
2807 2866 * added complications are for backwards compatibility
2808 2867 */
2809 PyObject *parse_index2(PyObject *self, PyObject *args)
2868 PyObject *parse_index2(PyObject *self, PyObject *args, PyObject *kwargs)
2810 2869 {
2811 2870 PyObject *cache = NULL;
2812 2871 indexObject *idx;
@@ -2816,7 +2875,7 b' PyObject *parse_index2(PyObject *self, P'
2816 2875 if (idx == NULL)
2817 2876 goto bail;
2818 2877
2819 ret = index_init(idx, args);
2878 ret = index_init(idx, args, kwargs);
2820 2879 if (ret == -1)
2821 2880 goto bail;
2822 2881
General Comments 0
You need to be logged in to leave comments. Login now