Merge pull request #7244 from minrk/rm-signature

don't store signatures in notebooks

IPython/html/services/contents/tests/test_manager.py

@@ -3,6 +3,7 b''
 from __future__ import print_function
 
 import os
+import time
 
 from tornado.web import HTTPError
 from unittest import TestCase
@@ -156,6 +157,7 b' class TestContentsManager(TestCase):'
 
         full_model = cm.get(path)
         nb = full_model['content']
+        nb['metadata']['counter'] = int(1e6 * time.time())
         self.add_code_cell(nb)
 
         cm.save(full_model, path)

IPython/nbformat/sign.py

@@ -1,18 +1,27 b''
-"""Functions for signing notebooks"""
+"""Utilities for signing notebooks"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import base64
 from contextlib import contextmanager
+from datetime import datetime
 import hashlib
 from hmac import HMAC
 import io
 import os
 
+try:
+    import sqlite3
+except ImportError:
+    try:
+        from pysqlite2 import dbapi2 as sqlite3
+    except ImportError:
+        sqlite3 = None
+
 from IPython.utils.io import atomic_writing
-from IPython.utils.py3compat import string_types, unicode_type, cast_bytes
-from IPython.utils.traitlets import Instance, Bytes, Enum, Any, Unicode, Bool
+from IPython.utils.py3compat import unicode_type, cast_bytes
+from IPython.utils.traitlets import Instance, Bytes, Enum, Any, Unicode, Bool, Integer
 from IPython.config import LoggingConfigurable, MultipleInstanceError
 from IPython.core.application import BaseIPythonApplication, base_flags
 
@@ -93,6 +102,48 b' class NotebookNotary(LoggingConfigurable):'
             app.initialize(argv=[])
         return app.profile_dir
     
+    db_file = Unicode(config=True,
+        help="""The sqlite file in which to store notebook signatures.
+        By default, this will be in your IPython profile.
+        You can set it to ':memory:' to disable sqlite writing to the filesystem.
+        """)
+    def _db_file_default(self):
+        if self.profile_dir is None:
+            return ':memory:'
+        return os.path.join(self.profile_dir.security_dir, u'nbsignatures.db')
+    
+    # 64k entries ~ 12MB
+    cache_size = Integer(65535, config=True,
+        help="""The number of notebook signatures to cache.
+        When the number of signatures exceeds this value,
+        the oldest 25% of signatures will be culled.
+        """
+    )
+    db = Any()
+    def _db_default(self):
+        if sqlite3 is None:
+            self.log.warn("Missing SQLite3, all notebooks will be untrusted!")
+            return
+        kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
+        db = sqlite3.connect(self.db_file, **kwargs)
+        self.init_db(db)
+        return db
+    
+    def init_db(self, db):
+        db.execute("""
+        CREATE TABLE IF NOT EXISTS nbsignatures
+        (
+            id integer PRIMARY KEY AUTOINCREMENT,
+            algorithm text,
+            signature text,
+            path text,
+            last_seen timestamp
+        )""")
+        db.execute("""
+        CREATE INDEX IF NOT EXISTS algosig ON nbsignatures(algorithm, signature)
+        """)
+        db.commit()
+    
     algorithm = Enum(algorithms, default_value='sha256', config=True,
         help="""The hashing algorithm used to sign notebooks."""
     )
@@ -168,28 +219,72 b' class NotebookNotary(LoggingConfigurable):'
         """
         if nb.nbformat < 3:
             return False
-        stored_signature = nb['metadata'].get('signature', None)
-        if not stored_signature \
-            or not isinstance(stored_signature, string_types) \
-            or ':' not in stored_signature:
+        if self.db is None:
             return False
-        stored_algo, sig = stored_signature.split(':', 1)
-        if self.algorithm != stored_algo:
+        signature = self.compute_signature(nb)
+        r = self.db.execute("""SELECT id FROM nbsignatures WHERE
+            algorithm = ? AND
+            signature = ?;
+            """, (self.algorithm, signature)).fetchone()
+        if r is None:
             return False
-        my_signature = self.compute_signature(nb)
-        return my_signature == sig
+        self.db.execute("""UPDATE nbsignatures SET last_seen = ? WHERE
+            algorithm = ? AND
+            signature = ?;
+            """,
+            (datetime.utcnow(), self.algorithm, signature),
+        )
+        self.db.commit()
+        return True
     
     def sign(self, nb):
-        """Sign a notebook, indicating that its output is trusted
-        
-        stores 'algo:hmac-hexdigest' in notebook.metadata.signature
+        """Sign a notebook, indicating that its output is trusted on this machine
         
-        e.g. 'sha256:deadbeef123...'
+        Stores hash algorithm and hmac digest in a local database of trusted notebooks.
         """
         if nb.nbformat < 3:
             return
         signature = self.compute_signature(nb)
-        nb['metadata']['signature'] = "%s:%s" % (self.algorithm, signature)
+        self.store_signature(signature, nb)
+
+    def store_signature(self, signature, nb):
+        if self.db is None:
+            return
+        self.db.execute("""INSERT OR IGNORE INTO nbsignatures
+            (algorithm, signature, last_seen) VALUES (?, ?, ?)""",
+            (self.algorithm, signature, datetime.utcnow())
+        )
+        self.db.execute("""UPDATE nbsignatures SET last_seen = ? WHERE
+            algorithm = ? AND
+            signature = ?;
+            """,
+            (datetime.utcnow(), self.algorithm, signature),
+        )
+        self.db.commit()
+        n, = self.db.execute("SELECT Count(*) FROM nbsignatures").fetchone()
+        if n > self.cache_size:
+            self.cull_db()
+    
+    def unsign(self, nb):
+        """Ensure that a notebook is untrusted
+        
+        by removing its signature from the trusted database, if present.
+        """
+        signature = self.compute_signature(nb)
+        self.db.execute("""DELETE FROM nbsignatures WHERE
+                algorithm = ? AND
+                signature = ?;
+            """,
+            (self.algorithm, signature)
+        )
+        self.db.commit()
+    
+    def cull_db(self):
+        """Cull oldest 25% of the trusted signatures when the size limit is reached"""
+        self.db.execute("""DELETE FROM nbsignatures WHERE id IN (
+            SELECT id FROM nbsignatures ORDER BY last_seen DESC LIMIT -1 OFFSET ?
+        );
+        """, (max(int(0.75 * self.cache_size), 1),))
     
     def mark_cells(self, nb, trusted):
         """Mark cells as trusted if the notebook's signature can be verified
@@ -222,11 +317,9 b' class NotebookNotary(LoggingConfigurable):'
         
         # explicitly safe output
         if nbformat_version >= 4:
-            safe = {'text/plain', 'image/png', 'image/jpeg'}
             unsafe_output_types = ['execute_result', 'display_data']
             safe_keys = {"output_type", "execution_count", "metadata"}
         else: # v3
-            safe = {'text', 'png', 'jpeg'}
             unsafe_output_types = ['pyout', 'display_data']
             safe_keys = {"output_type", "prompt_number", "metadata"}
         
@@ -261,7 +354,7 b' class NotebookNotary(LoggingConfigurable):'
 trust_flags = {
     'reset' : (
         {'TrustNotebookApp' : { 'reset' : True}},
-        """Generate a new key for notebook signature.
+        """Delete the trusted notebook cache.
         All previously signed notebooks will become untrusted.
         """
     ),
@@ -290,7 +383,7 b' class TrustNotebookApp(BaseIPythonApplication):'
     flags = trust_flags
     
     reset = Bool(False, config=True,
-        help="""If True, generate a new key for notebook signature.
+        help="""If True, delete the trusted signature cache.
         After reset, all previously signed notebooks will become untrusted.
         """
     )
@@ -320,6 +413,9 b' class TrustNotebookApp(BaseIPythonApplication):'
     
     def start(self):
         if self.reset:
+            if os.path.exists(self.notary.db_file):
+                print("Removing trusted signature cache: %s" % self.notary.db_file)
+                os.remove(self.notary.db_file)
             self.generate_new_key()
             return
         if not self.extra_args:

IPython/nbformat/tests/test_sign.py

@@ -3,6 +3,9 b''
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import copy
+import time
+
 from .base import TestsBase
 
 from IPython.nbformat import read, sign
@@ -13,8 +16,9 b' class TestNotary(TestsBase):'
     
     def setUp(self):
         self.notary = sign.NotebookNotary(
+            db_file=':memory:',
             secret=b'secret',
-            profile_dir=get_ipython().profile_dir
+            profile_dir=get_ipython().profile_dir,
         )
         with self.fopen(u'test3.ipynb', u'r') as f:
             self.nb = read(f, as_version=4)
@@ -25,10 +29,7 b' class TestNotary(TestsBase):'
         last_sig = ''
         for algo in sign.algorithms:
             self.notary.algorithm = algo
-            self.notary.sign(self.nb)
-            sig = self.nb.metadata.signature
-            print(sig)
-            self.assertEqual(sig[:len(self.notary.algorithm)+1], '%s:' % self.notary.algorithm)
+            sig = self.notary.compute_signature(self.nb)
             self.assertNotEqual(last_sig, sig)
             last_sig = sig
     
@@ -46,9 +47,49 b' class TestNotary(TestsBase):'
         self.assertNotEqual(sig1, sig2)
     
     def test_sign(self):
+        self.assertFalse(self.notary.check_signature(self.nb))
+        self.notary.sign(self.nb)
+        self.assertTrue(self.notary.check_signature(self.nb))
+    
+    def test_unsign(self):
         self.notary.sign(self.nb)
-        sig = self.nb.metadata.signature
-        self.assertEqual(sig[:len(self.notary.algorithm)+1], '%s:' % self.notary.algorithm)
+        self.assertTrue(self.notary.check_signature(self.nb))
+        self.notary.unsign(self.nb)
+        self.assertFalse(self.notary.check_signature(self.nb))
+        self.notary.unsign(self.nb)
+        self.assertFalse(self.notary.check_signature(self.nb))
+    
+    def test_cull_db(self):
+        # this test has various sleeps of 2ms
+        # to ensure low resolution timestamps compare as expected
+        dt = 2e-3
+        nbs = [
+            copy.deepcopy(self.nb) for i in range(10)
+        ]
+        for row in self.notary.db.execute("SELECT * FROM nbsignatures"):
+            print(row)
+        self.notary.cache_size = 8
+        for i, nb in enumerate(nbs[:8]):
+            nb.metadata.dirty = i
+            self.notary.sign(nb)
+        
+        for i, nb in enumerate(nbs[:8]):
+            time.sleep(dt)
+            self.assertTrue(self.notary.check_signature(nb), 'nb %i is trusted' % i)
+        
+        # signing the 9th triggers culling of first 3
+        # (75% of 8 = 6, 9 - 6 = 3 culled)
+        self.notary.sign(nbs[8])
+        self.assertFalse(self.notary.check_signature(nbs[0]))
+        self.assertFalse(self.notary.check_signature(nbs[1]))
+        self.assertFalse(self.notary.check_signature(nbs[2]))
+        self.assertTrue(self.notary.check_signature(nbs[3]))
+        # checking nb3 should keep it from being culled:
+        self.notary.sign(nbs[0])
+        self.notary.sign(nbs[1])
+        self.notary.sign(nbs[2])
+        self.assertTrue(self.notary.check_signature(nbs[3]))
+        self.assertFalse(self.notary.check_signature(nbs[4]))
     
     def test_check_signature(self):
         nb = self.nb

IPython/nbformat/v4/convert.py

@@ -56,6 +56,7 b' def upgrade(nb, from_version=3, from_minor=0):'
                 cells.append(upgrade_cell(cell))
         # upgrade metadata
         nb.metadata.pop('name', '')
+        nb.metadata.pop('signature', '')
         # Validate the converted notebook before returning it
         _warn_if_invalid(nb, nbformat)
         return nb

IPython/nbformat/v4/nbformat.v4.schema.json

@@ -55,10 +55,6 b''
                     }
                   }
                 },
-                "signature": {
-                    "description": "Hash of the notebook.",
-                    "type": "string"
-                },
                 "orig_nbformat": {
                     "description": "Original notebook format (major number) before converting the notebook between versions. This should never be written to a file.",
                     "type": "integer",

IPython/nbformat/v4/rwbase.py

@@ -64,6 +64,7 b' def strip_transient(nb):'
     """
     nb.metadata.pop('orig_nbformat', None)
     nb.metadata.pop('orig_nbformat_minor', None)
+    nb.metadata.pop('signature', None)
     for cell in nb.cells:
         cell.metadata.pop('trusted', None)
     return nb

examples/Customization/Index.ipynb

@@ -95,9 +95,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:de8cb1aff3da9097ba3fc7afab4327fc589f2bb1c154feeeb5ed97c87e37486f"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Embedding/Index.ipynb

@@ -186,9 +186,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:627cdf03b8de558c9344f9d1e8f0beeb2448e37e492d676e6db7b07d33251a2b"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/IPython Kernel/Beyond Plain Python.ipynb

@@ -1799,9 +1799,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:31071a05d0ecd75ed72fe3f0de0ad447a6f85cffe382c26efa5e68db1fee54ee"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/IPython Kernel/Capturing Output.ipynb

@@ -481,9 +481,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:df6354daf203e842bc040989d149760382d8ceec769160e4efe8cde9dfcb9107"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/IPython Kernel/Custom Display Logic.ipynb

@@ -1317,9 +1317,7 b''
    "source": []
   }
  ],
- "metadata": {
-  "signature": "sha256:86c779d5798c4a68bda7e71c8ef320cb7ba9d7e3d0f1bc4b828ee65f617a5ae3"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/IPython Kernel/Index.ipynb

@@ -160,9 +160,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:ee769d05a7e195e4b8546ef9a866ef03e59bff2f0fcba499d168c06b516aa79a"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/IPython Kernel/Plotting in the Notebook.ipynb

@@ -732,9 +732,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:74dbf5caa25c937be70dfe2ab509783a01f4a2044850d7044e729300a8c3644d"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/IPython Kernel/Raw Input in the Notebook.ipynb

@@ -150,9 +150,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:ac5c21534f3dd013c78d4d201527f3ed4dea5b6fad4116b8d23c67ba107e48c3"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/IPython Kernel/Rich Output.ipynb

@@ -3051,9 +3051,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:cf83dc9e6288480ac94c44a5983b4ee421f0ade792a9fac64bc00719263386c0"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }

  No newline at end of file

examples/IPython Kernel/SymPy.ipynb

@@ -1647,9 +1647,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:a0f4cda82587c5b8e7a4502192d1210abedac9084077604eb60aea15f262a344"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/IPython Kernel/Terminal Usage.ipynb

@@ -261,9 +261,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:993106eecfd7abe1920e1dbe670c4518189c26e7b29dcc541835f7dcf6fffbb2"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/IPython Kernel/Third Party Rich Output.ipynb

@@ -517,9 +517,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:123d82ef0551f78e5dca94db6e00f1e10ae07d930467cf44709ccc6a9216776a"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }

  No newline at end of file

examples/IPython Kernel/Trapezoid Rule.ipynb

@@ -374,9 +374,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:cbf49a9ceac0258773ae0a652e9ac7c13e0d042debbf0115cdc081862b5f7308"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/IPython Kernel/Working With External Code.ipynb

@@ -333,9 +333,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:4352d4e1c693d919ce40b29ecf5a536917160df68b19a85caccedb1ea7ad06e1"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Index.ipynb

@@ -41,9 +41,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:0b4b631419772e40e0f4893f5a0f0fe089a39e46c8862af0256164628394302d"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Interactive Widgets/Beat Frequencies.ipynb

@@ -591,9 +591,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:da6a3d73881e321e5ef406aea3e7d706c9dd405c94675318afde957d292f9cb9"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }

  No newline at end of file

examples/Interactive Widgets/Exploring Graphs.ipynb

@@ -830,9 +830,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:61f24f38b76cb12d46c178eadc5d85d61bd0fc27f959236ea756ef8e1adc2693"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Interactive Widgets/Factoring.ipynb

@@ -131,9 +131,7 b''
    "source": []
   }
  ],
- "metadata": {
-  "signature": "sha256:5f38c57d9570e4b6f6edf98c38a7bff81cd16365baa11fd40265f1504bfc008c"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Interactive Widgets/Image Browser.ipynb

@@ -297,9 +297,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:4e9f4ce8fa9be2e33ffdae399daec11bf3ef63a6f0065b1d59739310ebfe8008"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Interactive Widgets/Image Processing.ipynb

@@ -26762,9 +26762,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:d75ab1c53fa3389eeac78ecf8e89beb52871950f296aad25776699b6d6125037"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }

  No newline at end of file

examples/Interactive Widgets/Lorenz Differential Equations.ipynb

@@ -13428,9 +13428,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:c6ccfb14927d633933da8b5c0f776a8da756d833654b5a3f8b6121b390ca6424"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }

  No newline at end of file

examples/Notebook/Connecting with the Qt Console.ipynb

@@ -131,9 +131,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:9a1dd30de04270174c09ef33ca214e5b15ca3721547420087c1563ad557d78e3"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Notebook/Index.ipynb

@@ -68,9 +68,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:5a3c5ebae9154e13957e7c9d28bd3b7697c4b14f965482feb288d2cdf078983e"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Notebook/Notebook Security.ipynb

@@ -1,8 +1,6 b''
 {
  "cells": [],
- "metadata": {
-  "signature": "sha256:0abf067a20ebda26a671db997ac954770350d292dff7b7d6a4ace8808f70aca1"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Notebook/Running the Notebook Server.ipynb

@@ -355,9 +355,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:ee4b22b4c949fe21b3e5cda24f0916ba59d8c09443f4a897d98b96d4a73ac335"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Notebook/Working With Markdown Cells.ipynb

@@ -296,9 +296,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:3b7cae0c0936f25e6ccb7acafe310c08a4162a1a7fd66fa9874a52cffa0f64f9"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Parallel Computing/Index.ipynb

@@ -346,9 +346,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:1e9336d35cc07875300c5b876df6ce1f1971c2ee94870788c6ea32bbb789c42b"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file

examples/Parallel Computing/Monte Carlo Options.ipynb

@@ -2533,9 +2533,7 b''
    ]
   }
  ],
- "metadata": {
-  "signature": "sha256:1b19dedc6473d4e886e549020c6710f2d14c17296168a02e7e7fa9673912b893"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }

  No newline at end of file

examples/Parallel Computing/Parallel Decorator and map.ipynb

@@ -107,9 +107,7 b''
    "source": []
   }
  ],
- "metadata": {
-  "signature": "sha256:8781781d8835d77bf0f71b535b72ee2718405543c48e87ad0408242119cdb3cc"
- },
+ "metadata": {},
  "nbformat": 4,
  "nbformat_minor": 0
 }
  No newline at end of file