# HG changeset patch # User Raphaël Gomès # Date 2019-07-10 08:16:28 # Node ID 760a7851e9ba9c20bb0285237dc05adab46375af # Parent b3518b0baa477cb2ac2d82f90ef57a6e0bfa912f rust-parsers: move parser bindings to their own file and Python module This tidies up the Rust side while simplifying the Python side. Differential Revision: https://phab.mercurial-scm.org/D6627 diff --git a/mercurial/dirstate.py b/mercurial/dirstate.py --- a/mercurial/dirstate.py +++ b/mercurial/dirstate.py @@ -27,14 +27,14 @@ from . import ( util, ) -parsers = policy.importmod(r'parsers') -dirstatemod = policy.importrust(r'dirstate', default=parsers) +orig_parsers = policy.importmod(r'parsers') +parsers = policy.importrust(r'parsers', default=orig_parsers) propertycache = util.propertycache filecache = scmutil.filecache _rangemask = 0x7fffffff -dirstatetuple = parsers.dirstatetuple +dirstatetuple = orig_parsers.dirstatetuple class repocache(filecache): """filecache for files in .hg/""" @@ -1475,7 +1475,7 @@ class dirstatemap(object): # parsing the dirstate. # # (we cannot decorate the function directly since it is in a C module) - parse_dirstate = util.nogc(dirstatemod.parse_dirstate) + parse_dirstate = util.nogc(parsers.parse_dirstate) p = parse_dirstate(self._map, self.copymap, st) if not self._dirtyparents: self.setparents(*p) @@ -1486,8 +1486,8 @@ class dirstatemap(object): self.get = self._map.get def write(self, st, now): - st.write(dirstatemod.pack_dirstate(self._map, self.copymap, - self.parents(), now)) + st.write(parsers.pack_dirstate(self._map, self.copymap, + self.parents(), now)) st.close() self._dirtyparents = False self.nonnormalset, self.otherparentset = self.nonnormalentries() diff --git a/rust/hg-cpython/src/dirstate.rs b/rust/hg-cpython/src/dirstate.rs --- a/rust/hg-cpython/src/dirstate.rs +++ b/rust/hg-cpython/src/dirstate.rs @@ -12,19 +12,14 @@ mod dirs_multiset; use crate::dirstate::dirs_multiset::Dirs; use cpython::{ - exc, PyBytes, PyDict, PyErr, PyInt, PyModule, PyObject, PyResult, - PySequence, PyTuple, Python, PythonObject, ToPyObject, + PyBytes, PyDict, PyErr, PyModule, PyObject, PyResult, PySequence, Python, }; -use hg::{ - pack_dirstate, parse_dirstate, CopyVecEntry, DirstateEntry, - DirstatePackError, DirstateParents, DirstateParseError, DirstateVec, -}; +use hg::{DirstateEntry, DirstateVec}; use libc::{c_char, c_int}; #[cfg(feature = "python27")] use python27_sys::PyCapsule_Import; #[cfg(feature = "python3")] use python3_sys::PyCapsule_Import; -use std::collections::HashMap; use std::ffi::CStr; use std::mem::transmute; @@ -44,7 +39,9 @@ type MakeDirstateTupleFn = extern "C" fn /// This is largely a copy/paste from cindex.rs, pending the merge of a /// `py_capsule_fn!` macro in the rust-cpython project: /// https://github.com/dgrunwald/rust-cpython/pull/169 -fn decapsule_make_dirstate_tuple(py: Python) -> PyResult { +pub fn decapsule_make_dirstate_tuple( + py: Python, +) -> PyResult { unsafe { let caps_name = CStr::from_bytes_with_nul_unchecked( b"mercurial.cext.parsers.make_dirstate_tuple_CAPI\0", @@ -57,52 +54,7 @@ fn decapsule_make_dirstate_tuple(py: Pyt } } -fn parse_dirstate_wrapper( - py: Python, - dmap: PyDict, - copymap: PyDict, - st: PyBytes, -) -> PyResult { - match parse_dirstate(st.data(py)) { - Ok((parents, dirstate_vec, copies)) => { - for (filename, entry) in dirstate_vec { - dmap.set_item( - py, - PyBytes::new(py, &filename[..]), - decapsule_make_dirstate_tuple(py)?( - entry.state as c_char, - entry.mode, - entry.size, - entry.mtime, - ), - )?; - } - for CopyVecEntry { path, copy_path } in copies { - copymap.set_item( - py, - PyBytes::new(py, path), - PyBytes::new(py, copy_path), - )?; - } - Ok((PyBytes::new(py, parents.p1), PyBytes::new(py, parents.p2)) - .to_py_object(py)) - } - Err(e) => Err(PyErr::new::( - py, - match e { - DirstateParseError::TooLittleData => { - "too little data for parents".to_string() - } - DirstateParseError::Overflow => { - "overflow in dirstate".to_string() - } - DirstateParseError::CorruptedEntry(e) => e, - }, - )), - } -} - -fn extract_dirstate_vec( +pub fn extract_dirstate_vec( py: Python, dmap: &PyDict, ) -> Result { @@ -130,76 +82,6 @@ fn extract_dirstate_vec( .collect() } -fn pack_dirstate_wrapper( - py: Python, - dmap: PyDict, - copymap: PyDict, - pl: PyTuple, - now: PyInt, -) -> PyResult { - let p1 = pl.get_item(py, 0).extract::(py)?; - let p1: &[u8] = p1.data(py); - let p2 = pl.get_item(py, 1).extract::(py)?; - let p2: &[u8] = p2.data(py); - - let dirstate_vec = extract_dirstate_vec(py, &dmap)?; - - let copies: Result, Vec>, PyErr> = copymap - .items(py) - .iter() - .map(|(key, value)| { - Ok(( - key.extract::(py)?.data(py).to_owned(), - value.extract::(py)?.data(py).to_owned(), - )) - }) - .collect(); - - match pack_dirstate( - &dirstate_vec, - &copies?, - DirstateParents { p1, p2 }, - now.as_object().extract::(py)?, - ) { - Ok((packed, new_dirstate_vec)) => { - for ( - filename, - DirstateEntry { - state, - mode, - size, - mtime, - }, - ) in new_dirstate_vec - { - dmap.set_item( - py, - PyBytes::new(py, &filename[..]), - decapsule_make_dirstate_tuple(py)?( - state as c_char, - mode, - size, - mtime, - ), - )?; - } - Ok(PyBytes::new(py, &packed)) - } - Err(error) => Err(PyErr::new::( - py, - match error { - DirstatePackError::CorruptedParent => { - "expected a 20-byte hash".to_string() - } - DirstatePackError::CorruptedEntry(e) => e, - DirstatePackError::BadSize(expected, actual) => { - format!("bad dirstate size: {} != {}", actual, expected) - } - }, - )), - } -} - /// Create the module, with `__package__` given from parent pub fn init_module(py: Python, package: &str) -> PyResult { let dotted_name = &format!("{}.dirstate", package); @@ -207,27 +89,6 @@ pub fn init_module(py: Python, package: m.add(py, "__package__", package)?; m.add(py, "__doc__", "Dirstate - Rust implementation")?; - m.add( - py, - "parse_dirstate", - py_fn!( - py, - parse_dirstate_wrapper(dmap: PyDict, copymap: PyDict, st: PyBytes) - ), - )?; - m.add( - py, - "pack_dirstate", - py_fn!( - py, - pack_dirstate_wrapper( - dmap: PyDict, - copymap: PyDict, - pl: PyTuple, - now: PyInt - ) - ), - )?; m.add_class::(py)?; diff --git a/rust/hg-cpython/src/lib.rs b/rust/hg-cpython/src/lib.rs --- a/rust/hg-cpython/src/lib.rs +++ b/rust/hg-cpython/src/lib.rs @@ -29,6 +29,7 @@ mod cindex; mod conversion; pub mod dagops; pub mod dirstate; +pub mod parsers; pub mod discovery; pub mod exceptions; pub mod filepatterns; @@ -50,6 +51,11 @@ py_module_initializer!(rustext, initrust "filepatterns", filepatterns::init_module(py, &dotted_name)?, )?; + m.add( + py, + "parsers", + parsers::init_parsers_module(py, &dotted_name)?, + )?; m.add(py, "GraphError", py.get_type::())?; m.add( py, diff --git a/rust/hg-cpython/src/dirstate.rs b/rust/hg-cpython/src/parsers.rs copy from rust/hg-cpython/src/dirstate.rs copy to rust/hg-cpython/src/parsers.rs --- a/rust/hg-cpython/src/dirstate.rs +++ b/rust/hg-cpython/src/parsers.rs @@ -1,61 +1,28 @@ -// dirstate.rs +// parsers.rs // // Copyright 2019 Raphaël Gomès // // This software may be used and distributed according to the terms of the // GNU General Public License version 2 or any later version. -//! Bindings for the `hg::dirstate` module provided by the +//! Bindings for the `hg::dirstate::parsers` module provided by the //! `hg-core` package. //! -//! From Python, this will be seen as `mercurial.rustext.dirstate` -mod dirs_multiset; -use crate::dirstate::dirs_multiset::Dirs; +//! From Python, this will be seen as `mercurial.rustext.parsers` +//! use cpython::{ - exc, PyBytes, PyDict, PyErr, PyInt, PyModule, PyObject, PyResult, - PySequence, PyTuple, Python, PythonObject, ToPyObject, + exc, PyBytes, PyDict, PyErr, PyInt, PyModule, PyResult, PyTuple, Python, + PythonObject, ToPyObject, }; use hg::{ pack_dirstate, parse_dirstate, CopyVecEntry, DirstateEntry, - DirstatePackError, DirstateParents, DirstateParseError, DirstateVec, + DirstatePackError, DirstateParents, DirstateParseError, }; -use libc::{c_char, c_int}; -#[cfg(feature = "python27")] -use python27_sys::PyCapsule_Import; -#[cfg(feature = "python3")] -use python3_sys::PyCapsule_Import; use std::collections::HashMap; -use std::ffi::CStr; -use std::mem::transmute; -/// C code uses a custom `dirstate_tuple` type, checks in multiple instances -/// for this type, and raises a Python `Exception` if the check does not pass. -/// Because this type differs only in name from the regular Python tuple, it -/// would be a good idea in the near future to remove it entirely to allow -/// for a pure Python tuple of the same effective structure to be used, -/// rendering this type and the capsule below useless. -type MakeDirstateTupleFn = extern "C" fn( - state: c_char, - mode: c_int, - size: c_int, - mtime: c_int, -) -> PyObject; +use libc::c_char; -/// This is largely a copy/paste from cindex.rs, pending the merge of a -/// `py_capsule_fn!` macro in the rust-cpython project: -/// https://github.com/dgrunwald/rust-cpython/pull/169 -fn decapsule_make_dirstate_tuple(py: Python) -> PyResult { - unsafe { - let caps_name = CStr::from_bytes_with_nul_unchecked( - b"mercurial.cext.parsers.make_dirstate_tuple_CAPI\0", - ); - let from_caps = PyCapsule_Import(caps_name.as_ptr(), 0); - if from_caps.is_null() { - return Err(PyErr::fetch(py)); - } - Ok(transmute(from_caps)) - } -} +use crate::dirstate::{decapsule_make_dirstate_tuple, extract_dirstate_vec}; fn parse_dirstate_wrapper( py: Python, @@ -102,34 +69,6 @@ fn parse_dirstate_wrapper( } } -fn extract_dirstate_vec( - py: Python, - dmap: &PyDict, -) -> Result { - dmap.items(py) - .iter() - .map(|(filename, stats)| { - let stats = stats.extract::(py)?; - let state = stats.get_item(py, 0)?.extract::(py)?; - let state = state.data(py)[0] as i8; - let mode = stats.get_item(py, 1)?.extract(py)?; - let size = stats.get_item(py, 2)?.extract(py)?; - let mtime = stats.get_item(py, 3)?.extract(py)?; - let filename = filename.extract::(py)?; - let filename = filename.data(py); - Ok(( - filename.to_owned(), - DirstateEntry { - state, - mode, - size, - mtime, - }, - )) - }) - .collect() -} - fn pack_dirstate_wrapper( py: Python, dmap: PyDict, @@ -201,12 +140,13 @@ fn pack_dirstate_wrapper( } /// Create the module, with `__package__` given from parent -pub fn init_module(py: Python, package: &str) -> PyResult { - let dotted_name = &format!("{}.dirstate", package); +pub fn init_parsers_module(py: Python, package: &str) -> PyResult { + let dotted_name = &format!("{}.parsers", package); let m = PyModule::new(py, dotted_name)?; m.add(py, "__package__", package)?; - m.add(py, "__doc__", "Dirstate - Rust implementation")?; + m.add(py, "__doc__", "Parsers - Rust implementation")?; + m.add( py, "parse_dirstate", @@ -229,8 +169,6 @@ pub fn init_module(py: Python, package: ), )?; - m.add_class::(py)?; - let sys = PyModule::import(py, "sys")?; let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?; sys_modules.set_item(py, dotted_name, &m)?; diff --git a/tests/fakedirstatewritetime.py b/tests/fakedirstatewritetime.py --- a/tests/fakedirstatewritetime.py +++ b/tests/fakedirstatewritetime.py @@ -30,6 +30,7 @@ configitem(b'fakedirstatewritetime', b'f ) parsers = policy.importmod(r'parsers') +rustmod = policy.importrust(r'parsers') def pack_dirstate(fakenow, orig, dmap, copymap, pl, now): # execute what original parsers.pack_dirstate should do actually @@ -57,16 +58,21 @@ def fakewrite(ui, func): # 'fakenow' value and 'touch -t YYYYmmddHHMM' argument easy fakenow = dateutil.parsedate(fakenow, [b'%Y%m%d%H%M'])[0] - if rustext is not None: - orig_module = rustext.dirstate - orig_pack_dirstate = rustext.dirstate.pack_dirstate - else: - orig_module = parsers - orig_pack_dirstate = parsers.pack_dirstate + if rustmod is not None: + # The Rust implementation does not use public parse/pack dirstate + # to prevent conversion round-trips + orig_dirstatemap_write = dirstate.dirstatemap.write + wrapper = lambda self, st, now: orig_dirstatemap_write(self, + st, + fakenow) + dirstate.dirstatemap.write = wrapper orig_dirstate_getfsnow = dirstate._getfsnow wrapper = lambda *args: pack_dirstate(fakenow, orig_pack_dirstate, *args) + orig_module = parsers + orig_pack_dirstate = parsers.pack_dirstate + orig_module.pack_dirstate = wrapper dirstate._getfsnow = lambda *args: fakenow try: @@ -74,6 +80,8 @@ def fakewrite(ui, func): finally: orig_module.pack_dirstate = orig_pack_dirstate dirstate._getfsnow = orig_dirstate_getfsnow + if rustmod is not None: + dirstate.dirstatemap.write = orig_dirstatemap_write def _poststatusfixup(orig, workingctx, status, fixup): ui = workingctx.repo().ui