GITENV file updated 21

cgpenv/Lib/site-packages/spacy/pipeline/_parser_internals/_beam_utils.pxd

+from ...typedefs cimport class_t, hash_t
+
+# These are passed as callbacks to thinc.search.Beam
+cdef int transition_state(void* _dest, void* _src, class_t clas, void* _moves) except -1
+
+cdef int check_final_state(void* _state, void* extra_args) except -1

cgpenv/Lib/site-packages/spacy/pipeline/_parser_internals/arc_eager.pxd

+from ._state cimport StateC
+from ...typedefs cimport weight_t, attr_t
+from .transition_system cimport Transition, TransitionSystem
+
+
+cdef class ArcEager(TransitionSystem):
+    cdef get_arcs(self, StateC* state)

cgpenv/Lib/site-packages/spacy/pipeline/_parser_internals/ner.pxd

+from .transition_system cimport TransitionSystem
+
+
+cdef class BiluoPushDown(TransitionSystem):
+    pass

cgpenv/Lib/site-packages/spacy/pipeline/_parser_internals/nonproj.pyx

+# cython: profile=True, infer_types=True
+"""Implements the projectivize/deprojectivize mechanism in Nivre & Nilsson 2005
+for doing pseudo-projective parsing implementation uses the HEAD decoration
+scheme.
+"""
+from copy import copy
+
+from ...tokens.doc cimport Doc, set_children_from_heads
+
+from ...errors import Errors
+
+
+DELIMITER = '||'
+
+
+def ancestors(tokenid, heads):
+    # Returns all words going from the word up the path to the root. The path
+    # to root cannot be longer than the number of words in the  sentence. This
+    # function ends after at most len(heads) steps, because it would otherwise
+    # loop indefinitely on cycles.
+    head = tokenid
+    cnt = 0
+    while heads[head] != head and cnt < len(heads):
+        head = heads[head]
+        cnt += 1
+        yield head
+        if head is None:
+            break
+
+
+def contains_cycle(heads):
+    # in an acyclic tree, the path from each word following the head relation
+    # upwards always ends at the root node
+    for tokenid in range(len(heads)):
+        seen = set([tokenid])
+        for ancestor in ancestors(tokenid, heads):
+            if ancestor in seen:
+                return seen
+            seen.add(ancestor)
+    return None
+
+
+def is_nonproj_arc(tokenid, heads):
+    # definition (e.g. Havelka 2007): an arc h -> d, h < d is non-projective
+    # if there is a token k, h < k < d such that h is not
+    # an ancestor of k. Same for h -> d, h > d
+    head = heads[tokenid]
+    if head == tokenid:  # root arcs cannot be non-projective
+        return False
+    elif head is None:  # unattached tokens cannot be non-projective
+        return False
+    
+    cdef int start, end
+    if head < tokenid:
+        start, end = (head+1, tokenid)
+    else:
+        start, end = (tokenid+1, head)
+    for k in range(start, end):
+        for ancestor in ancestors(k, heads):
+            if ancestor is None:  # for unattached tokens/subtrees
+                break
+            elif ancestor == head:  # normal case: k dominated by h
+                break
+        else:  # head not in ancestors: d -> h is non-projective
+            return True
+    return False
+
+
+def is_nonproj_tree(heads):
+    # a tree is non-projective if at least one arc is non-projective
+    return any(is_nonproj_arc(word, heads) for word in range(len(heads)))
+
+
+def decompose(label):
+    return label.partition(DELIMITER)[::2]
+
+
+def is_decorated(label):
+    return DELIMITER in label
+
+def count_decorated_labels(gold_data):
+    freqs = {}
+    for example in gold_data:
+        proj_heads, deco_deps = projectivize(example.get_aligned("HEAD"),
+                                             example.get_aligned("DEP"))
+        # set the label to ROOT for each root dependent
+        deco_deps = ['ROOT' if head == i else deco_deps[i]
+                       for i, head in enumerate(proj_heads)]
+        # count label frequencies
+        for label in deco_deps:
+            if is_decorated(label):
+                freqs[label] = freqs.get(label, 0) + 1
+    return freqs
+
+
+def projectivize(heads, labels):
+    # Use the algorithm by Nivre & Nilsson 2005. Assumes heads to be a proper
+    # tree, i.e. connected and cycle-free. Returns a new pair (heads, labels)
+    # which encode a projective and decorated tree.
+    proj_heads = copy(heads)
+    smallest_np_arc = _get_smallest_nonproj_arc(proj_heads)
+    if smallest_np_arc is None:  # this sentence is already projective
+        return proj_heads, copy(labels)
+    while smallest_np_arc is not None:
+        _lift(smallest_np_arc, proj_heads)
+        smallest_np_arc = _get_smallest_nonproj_arc(proj_heads)
+    deco_labels = _decorate(heads, proj_heads, labels)
+    return proj_heads, deco_labels
+
+
+cpdef deprojectivize(Doc doc):
+    # Reattach arcs with decorated labels (following HEAD scheme). For each
+    # decorated arc X||Y, search top-down, left-to-right, breadth-first until
+    # hitting a Y then make this the new head.
+    for i in range(doc.length):
+        label = doc.vocab.strings[doc.c[i].dep]
+        if DELIMITER in label:
+            new_label, head_label = label.split(DELIMITER)
+            new_head = _find_new_head(doc[i], head_label)
+            doc.c[i].head = new_head.i - i
+            doc.c[i].dep = doc.vocab.strings.add(new_label)
+    set_children_from_heads(doc.c, 0, doc.length)
+    return doc
+
+
+def _decorate(heads, proj_heads, labels):
+    # uses decoration scheme HEAD from Nivre & Nilsson 2005
+    if (len(heads) != len(proj_heads)) or (len(proj_heads) != len(labels)):
+        raise ValueError(Errors.E082.format(n_heads=len(heads),
+                                            n_proj_heads=len(proj_heads),
+                                            n_labels=len(labels)))
+    deco_labels = []
+    for tokenid, head in enumerate(heads):
+        if head != proj_heads[tokenid]:
+            deco_labels.append(f"{labels[tokenid]}{DELIMITER}{labels[head]}")
+        else:
+            deco_labels.append(labels[tokenid])
+    return deco_labels
+
+
+def _get_smallest_nonproj_arc(heads):
+    # return the smallest non-proj arc or None
+    # where size is defined as the distance between dep and head
+    # and ties are broken left to right
+    smallest_size = float('inf')
+    smallest_np_arc = None
+    for tokenid, head in enumerate(heads):
+        size = abs(tokenid-head)
+        if size < smallest_size and is_nonproj_arc(tokenid, heads):
+            smallest_size = size
+            smallest_np_arc = tokenid
+    return smallest_np_arc
+
+
+def _lift(tokenid, heads):
+    # reattaches a word to it's grandfather
+    head = heads[tokenid]
+    ghead = heads[head]
+    # attach to ghead if head isn't attached to root else attach to root
+    heads[tokenid] = ghead if head != ghead else tokenid
+
+
+def _find_new_head(token, headlabel):
+    # search through the tree starting from the head of the given token
+    # returns the id of the first descendant with the given label
+    # if there is none, return the current head (no change)
+    queue = [token.head]
+    while queue:
+        next_queue = []
+        for qtoken in queue:
+            for child in qtoken.children:
+                if child.is_space:
+                    continue
+                if child == token:
+                    continue
+                if child.dep_ == headlabel:
+                    return child
+                next_queue.append(child)
+        queue = next_queue
+    return token.head

cgpenv/Lib/site-packages/spacy/pipeline/_parser_internals/stateclass.pxd

+from cymem.cymem cimport Pool
+
+from ...structs cimport TokenC, SpanC
+from ...typedefs cimport attr_t
+from ...tokens.doc cimport Doc
+
+from ._state cimport StateC
+
+
+cdef class StateClass:
+    cdef StateC* c
+    cdef readonly Doc doc
+    cdef int _borrowed
+
+    @staticmethod
+    cdef inline StateClass borrow(StateC* ptr, Doc doc):
+        cdef StateClass self = StateClass()
+        self.c = ptr
+        self._borrowed = 1
+        self.doc = doc
+        return self
+
+    @staticmethod
+    cdef inline StateClass init_offset(const TokenC* sent, int length, int
+                                       offset):
+        cdef StateClass self = StateClass()
+        self.c = new StateC(sent, length)
+        self.c.offset = offset
+        return self

cgpenv/Lib/site-packages/spacy/pipeline/_parser_internals/stateclass.pyx

+# cython: infer_types=True
+import numpy
+from libcpp.vector cimport vector
+from ._state cimport ArcC
+
+from ...tokens.doc cimport Doc
+
+
+cdef class StateClass:
+    def __init__(self, Doc doc=None, int offset=0):
+        self._borrowed = 0
+        if doc is not None:
+            self.c = new StateC(doc.c, doc.length)
+            self.c.offset = offset
+            self.doc = doc
+        else:
+            self.doc = None
+
+    def __dealloc__(self):
+        if self._borrowed != 1:
+            del self.c
+
+    @property
+    def stack(self):
+        return [self.S(i) for i in range(self.c.stack_depth())]
+
+    @property
+    def queue(self):
+        return [self.B(i) for i in range(self.c.buffer_length())]
+
+    @property
+    def token_vector_lenth(self):
+        return self.doc.tensor.shape[1]
+
+    @property
+    def arcs(self):
+        cdef vector[ArcC] arcs
+        self.c.get_arcs(&arcs)
+        return list(arcs)
+        #py_arcs = []
+        #for arc in arcs:
+        #    if arc.head != -1 and arc.child != -1:
+        #        py_arcs.append((arc.head, arc.child, arc.label))
+        #return arcs
+
+    def add_arc(self, int head, int child, int label):
+        self.c.add_arc(head, child, label)
+
+    def del_arc(self, int head, int child):
+        self.c.del_arc(head, child)
+
+    def H(self, int child):
+        return self.c.H(child)
+    
+    def L(self, int head, int idx):
+        return self.c.L(head, idx)
+    
+    def R(self, int head, int idx):
+        return self.c.R(head, idx)
+
+    @property
+    def _b_i(self):
+        return self.c._b_i
+
+    @property
+    def length(self):
+        return self.c.length
+
+    def is_final(self):
+        return self.c.is_final()
+
+    def copy(self):
+        cdef StateClass new_state = StateClass(doc=self.doc, offset=self.c.offset)
+        new_state.c.clone(self.c)
+        return new_state
+
+    def print_state(self):
+        words = [token.text for token in self.doc]
+        words = list(words) + ['_']
+        bools = ["F", "T"]
+        sent_starts = [bools[self.c.is_sent_start(i)] for i in range(len(self.doc))]
+        shifted = [1 if self.c.is_unshiftable(i) else 0 for i in range(self.c.length)]
+        shifted.append("")
+        sent_starts.append("")
+        top = f"{self.S(0)}{words[self.S(0)]}_{words[self.H(self.S(0))]}_{shifted[self.S(0)]}"
+        second = f"{self.S(1)}{words[self.S(1)]}_{words[self.H(self.S(1))]}_{shifted[self.S(1)]}"
+        third = f"{self.S(2)}{words[self.S(2)]}_{words[self.H(self.S(2))]}_{shifted[self.S(2)]}"
+        n0 = f"{self.B(0)}{words[self.B(0)]}_{sent_starts[self.B(0)]}_{shifted[self.B(0)]}"
+        n1 = f"{self.B(1)}{words[self.B(1)]}_{sent_starts[self.B(1)]}_{shifted[self.B(1)]}"
+        return ' '.join((str(self.stack_depth()), str(self.buffer_length()), third, second, top, '|', n0, n1))
+
+    def S(self, int i):
+        return self.c.S(i)
+
+    def B(self, int i):
+        return self.c.B(i)
+
+    def H(self, int i):
+        return self.c.H(i)
+    
+    def E(self, int i):
+        return self.c.E(i)
+
+    def L(self, int i, int idx):
+        return self.c.L(i, idx)
+
+    def R(self, int i, int idx):
+        return self.c.R(i, idx)
+
+    def S_(self, int i):
+        return self.doc[self.c.S(i)]
+
+    def B_(self, int i):
+        return self.doc[self.c.B(i)]
+
+    def H_(self, int i):
+        return self.doc[self.c.H(i)]
+    
+    def E_(self, int i):
+        return self.doc[self.c.E(i)]
+
+    def L_(self, int i, int idx):
+        return self.doc[self.c.L(i, idx)]
+
+    def R_(self, int i, int idx):
+        return self.doc[self.c.R(i, idx)]
+ 
+    def empty(self):
+        return self.c.empty()
+
+    def eol(self):
+        return self.c.eol()
+
+    def at_break(self):
+        return False
+        #return self.c.at_break()
+
+    def has_head(self, int i):
+        return self.c.has_head(i)
+
+    def  n_L(self, int i):
+        return self.c.n_L(i)
+
+    def n_R(self, int i):
+        return self.c.n_R(i)
+
+    def entity_is_open(self):
+        return self.c.entity_is_open()
+
+    def stack_depth(self):
+        return self.c.stack_depth()
+
+    def buffer_length(self):
+        return self.c.buffer_length()
+
+    def push(self):
+        self.c.push()
+
+    def pop(self):
+        self.c.pop()
+
+    def unshift(self):
+        self.c.unshift()
+
+    def add_arc(self, int head, int child, attr_t label):
+        self.c.add_arc(head, child, label)
+
+    def del_arc(self, int head, int child):
+        self.c.del_arc(head, child)
+
+    def open_ent(self, attr_t label):
+        self.c.open_ent(label)
+
+    def close_ent(self):
+        self.c.close_ent()
+
+    def clone(self, StateClass src):
+        self.c.clone(src.c)

cgpenv/Lib/site-packages/spacy/pipeline/_parser_internals/transition_system.pxd

+from cymem.cymem cimport Pool
+
+from ...typedefs cimport attr_t, weight_t
+from ...structs cimport TokenC
+from ...strings cimport StringStore
+from ...training.example cimport Example
+from .stateclass cimport StateClass
+from ._state cimport StateC
+
+
+cdef struct Transition:
+    int clas
+    int move
+    attr_t label
+
+    weight_t score
+
+    bint (*is_valid)(const StateC* state, attr_t label) nogil
+    weight_t (*get_cost)(const StateC* state, const void* gold, attr_t label) nogil
+    int (*do)(StateC* state, attr_t label) nogil
+
+
+ctypedef weight_t (*get_cost_func_t)(const StateC* state, const void* gold,
+        attr_tlabel) nogil
+ctypedef weight_t (*move_cost_func_t)(const StateC* state, const void* gold) nogil
+ctypedef weight_t (*label_cost_func_t)(const StateC* state, const void*
+        gold, attr_t label) nogil
+
+ctypedef int (*do_func_t)(StateC* state, attr_t label) nogil
+
+ctypedef void* (*init_state_t)(Pool mem, int length, void* tokens) except NULL
+
+ctypedef int (*del_state_t)(Pool mem, void* state, void* extra_args) except -1
+
+cdef class TransitionSystem:
+    cdef Pool mem
+    cdef StringStore strings
+    cdef Transition* c
+    cdef readonly int n_moves
+    cdef int _size
+    cdef public attr_t root_label
+    cdef public freqs
+    cdef public object labels
+    cdef public object cfg
+    cdef init_state_t init_beam_state
+    cdef del_state_t del_beam_state
+
+    cdef Transition lookup_transition(self, object name) except *
+
+    cdef Transition init_transition(self, int clas, int move, attr_t label) except *
+
+    cdef int set_valid(self, int* output, const StateC* st) nogil
+
+    cdef int set_costs(self, int* is_valid, weight_t* costs,
+                       const StateC* state, gold) except -1

cgpenv/Lib/site-packages/spacy/pipeline/_parser_internals/transition_system.pyx

+# cython: infer_types=True
+from __future__ import print_function
+from cymem.cymem cimport Pool
+
+from collections import Counter
+import srsly
+
+from . cimport _beam_utils
+from ...typedefs cimport weight_t, attr_t
+from ...tokens.doc cimport Doc
+from ...structs cimport TokenC
+from .stateclass cimport StateClass
+
+from ...errors import Errors
+from ... import util
+
+
+cdef weight_t MIN_SCORE = -90000
+
+
+class OracleError(Exception):
+    pass
+
+
+cdef void* _init_state(Pool mem, int length, void* tokens) except NULL:
+    cdef StateC* st = new StateC(<const TokenC*>tokens, length)
+    return <void*>st
+
+
+cdef int _del_state(Pool mem, void* state, void* x) except -1:
+    cdef StateC* st = <StateC*>state
+    del st
+
+
+cdef class TransitionSystem:
+    def __init__(
+        self,
+        StringStore string_table,
+        labels_by_action=None,
+        min_freq=None,
+        incorrect_spans_key=None
+    ):
+        self.cfg = {"neg_key": incorrect_spans_key}
+        self.mem = Pool()
+        self.strings = string_table
+        self.n_moves = 0
+        self._size = 100
+
+        self.c = <Transition*>self.mem.alloc(self._size, sizeof(Transition))
+
+        self.labels = {}
+        if labels_by_action:
+            self.initialize_actions(labels_by_action, min_freq=min_freq)
+        self.root_label = self.strings.add('ROOT')
+        self.init_beam_state = _init_state
+        self.del_beam_state = _del_state
+
+    def __reduce__(self):
+        # TODO: This loses the 'cfg'
+        return (self.__class__, (self.strings, self.labels), None, None)
+
+    @property
+    def neg_key(self):
+        return self.cfg.get("neg_key")
+
+    def init_batch(self, docs):
+        cdef StateClass state
+        states = []
+        offset = 0
+        for doc in docs:
+            state = StateClass(doc, offset=offset)
+            states.append(state)
+            offset += len(doc)
+        return states
+
+    def get_oracle_sequence(self, Example example, _debug=False):
+        states, golds, _ = self.init_gold_batch([example])
+        if not states:
+            return []
+        state = states[0]
+        gold = golds[0]
+        if _debug:
+            return self.get_oracle_sequence_from_state(state, gold, _debug=example)
+        else:
+            return self.get_oracle_sequence_from_state(state, gold)
+
+    def get_oracle_sequence_from_state(self, StateClass state, gold, _debug=None):
+        cdef Pool mem = Pool()
+        # n_moves should not be zero at this point, but make sure to avoid zero-length mem alloc
+        assert self.n_moves > 0
+        costs = <float*>mem.alloc(self.n_moves, sizeof(float))
+        is_valid = <int*>mem.alloc(self.n_moves, sizeof(int))
+
+        history = []
+        debug_log = []
+        while not state.is_final():
+            self.set_costs(is_valid, costs, state.c, gold)
+            for i in range(self.n_moves):
+                if is_valid[i] and costs[i] <= 0:
+                    action = self.c[i]
+                    history.append(i)
+                    if _debug:
+                        s0 = state.S(0)
+                        b0 = state.B(0)
+                        example = _debug
+                        debug_log.append(" ".join((
+                            self.get_class_name(i),
+                            "S0=", (example.x[s0].text if s0 >= 0 else "__"),
+                            "B0=", (example.x[b0].text if b0 >= 0 else "__"),
+                            "S0 head?", str(state.has_head(state.S(0))),
+                        )))
+                    action.do(state.c, action.label)
+                    break
+            else:
+                if _debug:
+                    example = _debug
+                    print("Actions")
+                    for i in range(self.n_moves):
+                        print(self.get_class_name(i))
+                    print("Gold")
+                    for token in example.y:
+                        print(token.text, token.dep_, token.head.text)
+                    s0 = state.S(0)
+                    b0 = state.B(0)
+                    debug_log.append(" ".join((
+                        "?",
+                        "S0=", (example.x[s0].text if s0 >= 0 else "-"),
+                        "B0=", (example.x[b0].text if b0 >= 0 else "-"),
+                        "S0 head?", str(state.has_head(state.S(0))),
+                    )))
+                    print("\n".join(debug_log))
+                raise ValueError(Errors.E024)
+        return history
+
+    def apply_transition(self, StateClass state, name):
+        if not self.is_valid(state, name):
+            raise ValueError(Errors.E170.format(name=name))
+        action = self.lookup_transition(name)
+        action.do(state.c, action.label)
+
+    cdef Transition lookup_transition(self, object name) except *:
+        raise NotImplementedError
+
+    cdef Transition init_transition(self, int clas, int move, attr_t label) except *:
+        raise NotImplementedError
+
+    def is_valid(self, StateClass stcls, move_name):
+        action = self.lookup_transition(move_name)
+        return action.is_valid(stcls.c, action.label)
+
+    cdef int set_valid(self, int* is_valid, const StateC* st) nogil:
+        cdef int i
+        for i in range(self.n_moves):
+            is_valid[i] = self.c[i].is_valid(st, self.c[i].label)
+
+    cdef int set_costs(self, int* is_valid, weight_t* costs,
+                       const StateC* state, gold) except -1:
+        raise NotImplementedError
+
+    def get_class_name(self, int clas):
+        act = self.c[clas]
+        return self.move_name(act.move, act.label)
+
+    def initialize_actions(self, labels_by_action, min_freq=None):
+        self.labels = {}
+        self.n_moves = 0
+        added_labels = []
+        added_actions = {}
+        for action, label_freqs in sorted(labels_by_action.items()):
+            action = int(action)
+            # Make sure we take a copy here, and that we get a Counter
+            self.labels[action] = Counter()
+            # Have to be careful here: Sorting must be stable, or our model
+            # won't be read back in correctly.
+            sorted_labels = [(f, L) for L, f in label_freqs.items()]
+            sorted_labels.sort()
+            sorted_labels.reverse()
+            for freq, label_str in sorted_labels:
+                if freq < 0:
+                    added_labels.append((freq, label_str))
+                    added_actions.setdefault(label_str, []).append(action)
+                else:
+                    self.add_action(int(action), label_str)
+                    self.labels[action][label_str] = freq
+        added_labels.sort(reverse=True)
+        for freq, label_str in added_labels:
+            for action in added_actions[label_str]:
+                self.add_action(int(action), label_str)
+                self.labels[action][label_str] = freq
+
+    def add_action(self, int action, label_name):
+        cdef attr_t label_id
+        if not isinstance(label_name, int) and \
+           not isinstance(label_name, long):
+            label_id = self.strings.add(label_name)
+        else:
+            label_id = label_name
+        # Check we're not creating a move we already have, so that this is
+        # idempotent
+        for trans in self.c[:self.n_moves]:
+            if trans.move == action and trans.label == label_id:
+                return 0
+        if self.n_moves >= self._size:
+            self._size *= 2
+            self.c = <Transition*>self.mem.realloc(self.c, self._size * sizeof(self.c[0]))
+        self.c[self.n_moves] = self.init_transition(self.n_moves, action, label_id)
+        self.n_moves += 1
+        # Add the new (action, label) pair, making up a frequency for it if
+        # necessary. To preserve sort order, the frequency needs to be lower
+        # than previous frequencies.
+        if self.labels.get(action, []):
+            new_freq = min(self.labels[action].values())
+        else:
+            self.labels[action] = Counter()
+            new_freq = -1
+        if new_freq > 0:
+            new_freq = 0
+        self.labels[action][label_name] = new_freq-1
+        return 1
+
+    def to_disk(self, path, **kwargs):
+        with path.open('wb') as file_:
+            file_.write(self.to_bytes(**kwargs))
+
+    def from_disk(self, path, **kwargs):
+        with path.open('rb') as file_:
+            byte_data = file_.read()
+        self.from_bytes(byte_data, **kwargs)
+        return self
+
+    def to_bytes(self, exclude=tuple()):
+        transitions = []
+        serializers = {
+            'moves': lambda: srsly.json_dumps(self.labels),
+            'strings': lambda: self.strings.to_bytes(),
+            'cfg': lambda: self.cfg
+        }
+        return util.to_bytes(serializers, exclude)
+
+    def from_bytes(self, bytes_data, exclude=tuple()):
+        # We're adding a new field, 'cfg', here and we don't want to break
+        # previous models that don't have it.
+        msg = srsly.msgpack_loads(bytes_data)
+        labels = {}
+        if 'moves' not in exclude:
+            labels.update(srsly.json_loads(msg['moves']))
+        if 'strings' not in exclude:
+            self.strings.from_bytes(msg['strings'])
+        if 'cfg' not in exclude and 'cfg' in msg:
+            self.cfg.update(msg['cfg'])
+        self.initialize_actions(labels)
+        return self