rust-parsers: move parser bindings to their own file and Python module

This tidies up the Rust side while simplifying the Python side.

Differential Revision: https://phab.mercurial-scm.org/D6627

mercurial/dirstate.py

@@ -27,14 +27,14 b' from . import ('
     util,
 )
 
-parsers = policy.importmod(r'parsers')
-dirstatemod = policy.importrust(r'dirstate', default=parsers)
+orig_parsers = policy.importmod(r'parsers')
+parsers = policy.importrust(r'parsers', default=orig_parsers)
 
 propertycache = util.propertycache
 filecache = scmutil.filecache
 _rangemask = 0x7fffffff
 
-dirstatetuple = parsers.dirstatetuple
+dirstatetuple = orig_parsers.dirstatetuple
 
 class repocache(filecache):
     """filecache for files in .hg/"""
@@ -1475,7 +1475,7 b' class dirstatemap(object):'
         # parsing the dirstate.
         #
         # (we cannot decorate the function directly since it is in a C module)
-        parse_dirstate = util.nogc(dirstatemod.parse_dirstate)
+        parse_dirstate = util.nogc(parsers.parse_dirstate)
         p = parse_dirstate(self._map, self.copymap, st)
         if not self._dirtyparents:
             self.setparents(*p)
@@ -1486,8 +1486,8 b' class dirstatemap(object):'
         self.get = self._map.get
 
     def write(self, st, now):
-        st.write(dirstatemod.pack_dirstate(self._map, self.copymap,
-                                           self.parents(), now))
+        st.write(parsers.pack_dirstate(self._map, self.copymap,
+                                       self.parents(), now))
         st.close()
         self._dirtyparents = False
         self.nonnormalset, self.otherparentset = self.nonnormalentries()

rust/hg-cpython/src/dirstate.rs

@@ -12,19 +12,14 b''
 mod dirs_multiset;
 use crate::dirstate::dirs_multiset::Dirs;
 use cpython::{
-    exc, PyBytes, PyDict, PyErr, PyInt, PyModule, PyObject, PyResult,
-    PySequence, PyTuple, Python, PythonObject, ToPyObject,
+    PyBytes, PyDict, PyErr, PyModule, PyObject, PyResult, PySequence, Python,
 };
-use hg::{
-    pack_dirstate, parse_dirstate, CopyVecEntry, DirstateEntry,
-    DirstatePackError, DirstateParents, DirstateParseError, DirstateVec,
-};
+use hg::{DirstateEntry, DirstateVec};
 use libc::{c_char, c_int};
 #[cfg(feature = "python27")]
 use python27_sys::PyCapsule_Import;
 #[cfg(feature = "python3")]
 use python3_sys::PyCapsule_Import;
-use std::collections::HashMap;
 use std::ffi::CStr;
 use std::mem::transmute;
 
@@ -44,7 +39,9 b' type MakeDirstateTupleFn = extern "C" fn'
 /// This is largely a copy/paste from cindex.rs, pending the merge of a
 /// `py_capsule_fn!` macro in the rust-cpython project:
 /// https://github.com/dgrunwald/rust-cpython/pull/169
-fn decapsule_make_dirstate_tuple(py: Python) -> PyResult<MakeDirstateTupleFn> {
+pub fn decapsule_make_dirstate_tuple(
+    py: Python,
+) -> PyResult<MakeDirstateTupleFn> {
     unsafe {
         let caps_name = CStr::from_bytes_with_nul_unchecked(
             b"mercurial.cext.parsers.make_dirstate_tuple_CAPI\0",
@@ -57,52 +54,7 b' fn decapsule_make_dirstate_tuple(py: Pyt'
     }
 }
 
-fn parse_dirstate_wrapper(
-    py: Python,
-    dmap: PyDict,
-    copymap: PyDict,
-    st: PyBytes,
-) -> PyResult<PyTuple> {
-    match parse_dirstate(st.data(py)) {
-        Ok((parents, dirstate_vec, copies)) => {
-            for (filename, entry) in dirstate_vec {
-                dmap.set_item(
-                    py,
-                    PyBytes::new(py, &filename[..]),
-                    decapsule_make_dirstate_tuple(py)?(
-                        entry.state as c_char,
-                        entry.mode,
-                        entry.size,
-                        entry.mtime,
-                    ),
-                )?;
-            }
-            for CopyVecEntry { path, copy_path } in copies {
-                copymap.set_item(
-                    py,
-                    PyBytes::new(py, path),
-                    PyBytes::new(py, copy_path),
-                )?;
-            }
-            Ok((PyBytes::new(py, parents.p1), PyBytes::new(py, parents.p2))
-                .to_py_object(py))
-        }
-        Err(e) => Err(PyErr::new::<exc::ValueError, _>(
-            py,
-            match e {
-                DirstateParseError::TooLittleData => {
-                    "too little data for parents".to_string()
-                }
-                DirstateParseError::Overflow => {
-                    "overflow in dirstate".to_string()
-                }
-                DirstateParseError::CorruptedEntry(e) => e,
-            },
-        )),
-    }
-}
-
-fn extract_dirstate_vec(
+pub fn extract_dirstate_vec(
     py: Python,
     dmap: &PyDict,
 ) -> Result<DirstateVec, PyErr> {
@@ -130,76 +82,6 b' fn extract_dirstate_vec('
         .collect()
 }
 
-fn pack_dirstate_wrapper(
-    py: Python,
-    dmap: PyDict,
-    copymap: PyDict,
-    pl: PyTuple,
-    now: PyInt,
-) -> PyResult<PyBytes> {
-    let p1 = pl.get_item(py, 0).extract::<PyBytes>(py)?;
-    let p1: &[u8] = p1.data(py);
-    let p2 = pl.get_item(py, 1).extract::<PyBytes>(py)?;
-    let p2: &[u8] = p2.data(py);
-
-    let dirstate_vec = extract_dirstate_vec(py, &dmap)?;
-
-    let copies: Result<HashMap<Vec<u8>, Vec<u8>>, PyErr> = copymap
-        .items(py)
-        .iter()
-        .map(|(key, value)| {
-            Ok((
-                key.extract::<PyBytes>(py)?.data(py).to_owned(),
-                value.extract::<PyBytes>(py)?.data(py).to_owned(),
-            ))
-        })
-        .collect();
-
-    match pack_dirstate(
-        &dirstate_vec,
-        &copies?,
-        DirstateParents { p1, p2 },
-        now.as_object().extract::<i32>(py)?,
-    ) {
-        Ok((packed, new_dirstate_vec)) => {
-            for (
-                filename,
-                DirstateEntry {
-                    state,
-                    mode,
-                    size,
-                    mtime,
-                },
-            ) in new_dirstate_vec
-            {
-                dmap.set_item(
-                    py,
-                    PyBytes::new(py, &filename[..]),
-                    decapsule_make_dirstate_tuple(py)?(
-                        state as c_char,
-                        mode,
-                        size,
-                        mtime,
-                    ),
-                )?;
-            }
-            Ok(PyBytes::new(py, &packed))
-        }
-        Err(error) => Err(PyErr::new::<exc::ValueError, _>(
-            py,
-            match error {
-                DirstatePackError::CorruptedParent => {
-                    "expected a 20-byte hash".to_string()
-                }
-                DirstatePackError::CorruptedEntry(e) => e,
-                DirstatePackError::BadSize(expected, actual) => {
-                    format!("bad dirstate size: {} != {}", actual, expected)
-                }
-            },
-        )),
-    }
-}
-
 /// Create the module, with `__package__` given from parent
 pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> {
     let dotted_name = &format!("{}.dirstate", package);
@@ -207,27 +89,6 b' pub fn init_module(py: Python, package: '
 
     m.add(py, "__package__", package)?;
     m.add(py, "__doc__", "Dirstate - Rust implementation")?;
-    m.add(
-        py,
-        "parse_dirstate",
-        py_fn!(
-            py,
-            parse_dirstate_wrapper(dmap: PyDict, copymap: PyDict, st: PyBytes)
-        ),
-    )?;
-    m.add(
-        py,
-        "pack_dirstate",
-        py_fn!(
-            py,
-            pack_dirstate_wrapper(
-                dmap: PyDict,
-                copymap: PyDict,
-                pl: PyTuple,
-                now: PyInt
-            )
-        ),
-    )?;
 
     m.add_class::<Dirs>(py)?;
 

rust/hg-cpython/src/lib.rs

@@ -29,6 +29,7 b' mod cindex;'
 mod conversion;
 pub mod dagops;
 pub mod dirstate;
+pub mod parsers;
 pub mod discovery;
 pub mod exceptions;
 pub mod filepatterns;
@@ -50,6 +51,11 b' py_module_initializer!(rustext, initrust'
         "filepatterns",
         filepatterns::init_module(py, &dotted_name)?,
     )?;
+    m.add(
+        py,
+        "parsers",
+        parsers::init_parsers_module(py, &dotted_name)?,
+    )?;
     m.add(py, "GraphError", py.get_type::<exceptions::GraphError>())?;
     m.add(
         py,

rust/hg-cpython/src/parsers.rs

@@ -1,61 +1,28 b''
-// dirstate.rs
+// parsers.rs
 //
 // Copyright 2019 Raphaël Gomès <[e-mail]>
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
-//! Bindings for the `hg::dirstate` module provided by the
+//! Bindings for the `hg::dirstate::parsers` module provided by the
 //! `hg-core` package.
 //!
-//! From Python, this will be seen as `mercurial.rustext.dirstate`
-mod dirs_multiset;
-use crate::dirstate::dirs_multiset::Dirs;
+//! From Python, this will be seen as `mercurial.rustext.parsers`
+//!
 use cpython::{
-    exc, PyBytes, PyDict, PyErr, PyInt, PyModule, PyObject, PyResult,
-    PySequence, PyTuple, Python, PythonObject, ToPyObject,
+    exc, PyBytes, PyDict, PyErr, PyInt, PyModule, PyResult, PyTuple, Python,
+    PythonObject, ToPyObject,
 };
 use hg::{
     pack_dirstate, parse_dirstate, CopyVecEntry, DirstateEntry,
-    DirstatePackError, DirstateParents, DirstateParseError, DirstateVec,
+    DirstatePackError, DirstateParents, DirstateParseError,
 };
-use libc::{c_char, c_int};
-#[cfg(feature = "python27")]
-use python27_sys::PyCapsule_Import;
-#[cfg(feature = "python3")]
-use python3_sys::PyCapsule_Import;
 use std::collections::HashMap;
-use std::ffi::CStr;
-use std::mem::transmute;
 
-/// C code uses a custom `dirstate_tuple` type, checks in multiple instances
-/// for this type, and raises a Python `Exception` if the check does not pass.
-/// Because this type differs only in name from the regular Python tuple, it
-/// would be a good idea in the near future to remove it entirely to allow
-/// for a pure Python tuple of the same effective structure to be used,
-/// rendering this type and the capsule below useless.
-type MakeDirstateTupleFn = extern "C" fn(
-    state: c_char,
-    mode: c_int,
-    size: c_int,
-    mtime: c_int,
-) -> PyObject;
+use libc::c_char;
 
-/// This is largely a copy/paste from cindex.rs, pending the merge of a
-/// `py_capsule_fn!` macro in the rust-cpython project:
-/// https://github.com/dgrunwald/rust-cpython/pull/169
-fn decapsule_make_dirstate_tuple(py: Python) -> PyResult<MakeDirstateTupleFn> {
-    unsafe {
-        let caps_name = CStr::from_bytes_with_nul_unchecked(
-            b"mercurial.cext.parsers.make_dirstate_tuple_CAPI\0",
-        );
-        let from_caps = PyCapsule_Import(caps_name.as_ptr(), 0);
-        if from_caps.is_null() {
-            return Err(PyErr::fetch(py));
-        }
-        Ok(transmute(from_caps))
-    }
-}
+use crate::dirstate::{decapsule_make_dirstate_tuple, extract_dirstate_vec};
 
 fn parse_dirstate_wrapper(
     py: Python,
@@ -102,34 +69,6 b' fn parse_dirstate_wrapper('
     }
 }
 
-fn extract_dirstate_vec(
-    py: Python,
-    dmap: &PyDict,
-) -> Result<DirstateVec, PyErr> {
-    dmap.items(py)
-        .iter()
-        .map(|(filename, stats)| {
-            let stats = stats.extract::<PySequence>(py)?;
-            let state = stats.get_item(py, 0)?.extract::<PyBytes>(py)?;
-            let state = state.data(py)[0] as i8;
-            let mode = stats.get_item(py, 1)?.extract(py)?;
-            let size = stats.get_item(py, 2)?.extract(py)?;
-            let mtime = stats.get_item(py, 3)?.extract(py)?;
-            let filename = filename.extract::<PyBytes>(py)?;
-            let filename = filename.data(py);
-            Ok((
-                filename.to_owned(),
-                DirstateEntry {
-                    state,
-                    mode,
-                    size,
-                    mtime,
-                },
-            ))
-        })
-        .collect()
-}
-
 fn pack_dirstate_wrapper(
     py: Python,
     dmap: PyDict,
@@ -201,12 +140,13 b' fn pack_dirstate_wrapper('
 }
 
 /// Create the module, with `__package__` given from parent
-pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> {
-    let dotted_name = &format!("{}.dirstate", package);
+pub fn init_parsers_module(py: Python, package: &str) -> PyResult<PyModule> {
+    let dotted_name = &format!("{}.parsers", package);
     let m = PyModule::new(py, dotted_name)?;
 
     m.add(py, "__package__", package)?;
-    m.add(py, "__doc__", "Dirstate - Rust implementation")?;
+    m.add(py, "__doc__", "Parsers - Rust implementation")?;
+
     m.add(
         py,
         "parse_dirstate",
@@ -229,8 +169,6 b' pub fn init_module(py: Python, package: '
         ),
     )?;
 
-    m.add_class::<Dirs>(py)?;
-
     let sys = PyModule::import(py, "sys")?;
     let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?;
     sys_modules.set_item(py, dotted_name, &m)?;

tests/fakedirstatewritetime.py

@@ -30,6 +30,7 b" configitem(b'fakedirstatewritetime', b'f"
 )
 
 parsers = policy.importmod(r'parsers')
+rustmod = policy.importrust(r'parsers')
 
 def pack_dirstate(fakenow, orig, dmap, copymap, pl, now):
     # execute what original parsers.pack_dirstate should do actually
@@ -57,16 +58,21 b' def fakewrite(ui, func):'
     # 'fakenow' value and 'touch -t YYYYmmddHHMM' argument easy
     fakenow = dateutil.parsedate(fakenow, [b'%Y%m%d%H%M'])[0]
 
-    if rustext is not None:
-        orig_module = rustext.dirstate
-        orig_pack_dirstate = rustext.dirstate.pack_dirstate
-    else:
-        orig_module = parsers
-        orig_pack_dirstate = parsers.pack_dirstate
+    if rustmod is not None:
+        # The Rust implementation does not use public parse/pack dirstate
+        # to prevent conversion round-trips
+        orig_dirstatemap_write = dirstate.dirstatemap.write
+        wrapper = lambda self, st, now: orig_dirstatemap_write(self,
+                                                               st,
+                                                               fakenow)
+        dirstate.dirstatemap.write = wrapper
 
     orig_dirstate_getfsnow = dirstate._getfsnow
     wrapper = lambda *args: pack_dirstate(fakenow, orig_pack_dirstate, *args)
 
+    orig_module = parsers
+    orig_pack_dirstate = parsers.pack_dirstate
+
     orig_module.pack_dirstate = wrapper
     dirstate._getfsnow = lambda *args: fakenow
     try:
@@ -74,6 +80,8 b' def fakewrite(ui, func):'
     finally:
         orig_module.pack_dirstate = orig_pack_dirstate
         dirstate._getfsnow = orig_dirstate_getfsnow
+        if rustmod is not None:
+            dirstate.dirstatemap.write = orig_dirstatemap_write
 
 def _poststatusfixup(orig, workingctx, status, fixup):
     ui = workingctx.repo().ui