BVh(VdZddlmZddlmZddlmZddlmZddlmZ ddlm Z ddlm Z dd l m Z d Ze d Gd d e jZe ddddde j"ej$dfdZe ddddde j"ej$dfdZy)zLookup operations.)assert_cardinality)dtypes)ops)tensor)gen_experimental_dataset_ops) lookup_ops)math_ops) tf_exportc`d}d}t|dk7rt|dt|d|dzt|dtjst|dt |ddzt|d tjst|d t |d dz|dj jd vr)t|d |dj jdz|d j jd vr)t|d |d j jdzy)zGRaises an error if the given table initializer element spec is invalid.zDatasets used to initialize lookup tables must produce elements in the form (key, value), where the keys and values are scalar tensors. Nz$However, the given dataset produces zO components instead of two (key, value) components. Full dataset element spec: .rzHowever, the given dataset produces non-Tensor values of type )NrzCHowever, the given dataset produces non-scalar key Tensors of rank zEHowever, the given dataset produces non-scalar value Tensors of rank )len ValueError isinstancer TensorSpectypeshaperank) element_spec base_errorspecific_errors b/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/data/experimental/ops/lookup_ops.py%_check_table_initializer_element_specrs;*.! Z"HL)*+L$~Q#(( )) LOV%6%6 7 Z#004\!_0E/Fa#II JJ LOV%6%6 7 Z#226|A2G1H#KK LL!_y0 **6q/*?*?*D*D)EQH H II!_y0 ,,8O,A,A,F,F+GqJ J KK1z$data.experimental.DatasetInitializerc(eZdZdZfdZdZxZS)DatasetInitializeraCreates a table initializer from a `tf.data.Dataset`. Sample usage: >>> keys = tf.data.Dataset.range(100) >>> values = tf.data.Dataset.range(100).map( ... lambda x: tf.strings.as_string(x * 2)) >>> ds = tf.data.Dataset.zip((keys, values)) >>> init = tf.data.experimental.DatasetInitializer(ds) >>> table = tf.lookup.StaticHashTable(init, "") >>> table.lookup(tf.constant([0, 1, 2], dtype=tf.int64)).numpy() array([b'0', b'2', b'4'], dtype=object) Attributes: dataset: A `tf.data.Dataset` object that produces tuples of scalars. The first scalar is treated as a key and the second as value. Raises: ValueError if `dataset` doesn't conform to specifications. c||_|jj}t||dj}|dj}tt |||y)aHCreates a table initializer from a `tf.data.Dataset`. Args: dataset: A `tf.data.Dataset` object that produces tuples of scalars. The first scalar is treated as a key and the second as value. Raises: ValueError if `dataset` doesn't conform to specifications. Returns: A `DatasetInitializer` object rrN)datasetrrdtypesuperr__init__)selfr elem_speckey_type value_type __class__s rr"zDatasetInitializer.__init__KsSDL ))I))4|!!H1##J d,XzBrc tj||j|jt j |j |jj}tjtjj||SN) rcheck_table_dtypes _key_dtype _value_dtypeged_opsinitialize_table_from_datasetresource_handler_variant_tensorradd_to_collection GraphKeysTABLE_INITIALIZERS)r#tableinit_ops r initializezDatasetInitializer.initialize^sb!!%$:K:KL33 t||;;=G#--::GD Nr)__name__ __module__ __qualname____doc__r"r6 __classcell__)r's@rrr6s&C&rrz$data.experimental.table_from_datasetNc|j}t||Pd}|djjs5|djjst d|djd|dkrt d|dt |tjs||dkrt d|d|js+tj|jk7rtd ||it |tjr$tj|tj}|j!|}|j#t%|}t'j(|d 5t+|}t'j(dd 5t-j.||} |rt-j0| ||| } | cdddcdddS#1swYnxYw dddy#1swYyxYw) aReturns a lookup table based on the given dataset. This operation constructs a lookup table based on the given dataset of pairs of (key, value). Any lookup of an out-of-vocabulary token will return a bucket ID based on its hash if `num_oov_buckets` is greater than zero. Otherwise it is assigned the `default_value`. The bucket ID range is `[vocabulary size, vocabulary size + num_oov_buckets - 1]`. Sample Usages: >>> keys = tf.data.Dataset.range(100) >>> values = tf.data.Dataset.range(100).map( ... lambda x: tf.strings.as_string(x * 2)) >>> ds = tf.data.Dataset.zip((keys, values)) >>> table = tf.data.experimental.table_from_dataset( ... ds, default_value='n/a', key_dtype=tf.int64) >>> table.lookup(tf.constant([0, 1, 2], dtype=tf.int64)).numpy() array([b'0', b'2', b'4'], dtype=object) Args: dataset: A dataset containing (key, value) pairs. num_oov_buckets: The number of out-of-vocabulary buckets. vocab_size: Number of the elements in the vocabulary, if known. default_value: The value to use for out-of-vocabulary feature values. Defaults to -1. hasher_spec: A `HasherSpec` to specify the hash function to use for assignation of out-of-vocabulary buckets. key_dtype: The `key` data type. name: A name for this op (optional). Returns: The lookup table based on the given dataset. Raises: ValueError: If * `dataset` does not contain pairs * The 2nd item in the `dataset` pairs has a dtype which is incompatible with `default_value` * `num_oov_buckets` is negative * `vocab_size` is not greater than zero * The `key_dtype` is not integer or string Nrzd`default_value` must be specified when creating a table from a dataset that produces values of type r rz:`num_oov_buckets` must be greater than or equal to 0, got z)`vocab_size` must be greater than 0, got z>`key_dtype` must be either an integer or string type, but got string_to_index hash_table)num_oov_buckets hasher_spec key_dtype)rrr is_integer is_floatingrrrTensorrstring base_dtype TypeErrorr castint64takeapplyrr name_scoperrStaticHashTableV1IdTableWithHashBuckets) rr@ vocab_size default_valuerArBnamer$ initializerr4s rtable_from_datasetrTfsj"")' 2M aL   ) )Yq\-?-?-K-K L#A,,,-Q0 11q +,A/ 00 Z /J4J1n @ AN OO   V]]i6J6J%J (k+ ,,*fmm,==V\\:jll:&Gmm.z:;G ~~d-. $W-K l +**; Fe 11 +# !      s$&"G$3G; G$G G$$G-z*data.experimental.index_table_from_datasetr=c `t|jjd||||||S)aReturns an index lookup table based on the given dataset. This operation constructs a lookup table based on the given dataset of keys. Any lookup of an out-of-vocabulary token will return a bucket ID based on its hash if `num_oov_buckets` is greater than zero. Otherwise it is assigned the `default_value`. The bucket ID range is `[vocabulary size, vocabulary size + num_oov_buckets - 1]`. Sample Usages: >>> ds = tf.data.Dataset.range(100).map(lambda x: tf.strings.as_string(x * 2)) >>> table = tf.data.experimental.index_table_from_dataset( ... ds, key_dtype=dtypes.int64) >>> table.lookup(tf.constant(['0', '2', '4'], dtype=tf.string)).numpy() array([0, 1, 2]) Args: dataset: A dataset of keys. num_oov_buckets: The number of out-of-vocabulary buckets. vocab_size: Number of the elements in the vocabulary, if known. default_value: The value to use for out-of-vocabulary feature values. Defaults to -1. hasher_spec: A `HasherSpec` to specify the hash function to use for assignation of out-of-vocabulary buckets. key_dtype: The `key` data type. name: A name for this op (optional). Returns: The lookup table based on the given dataset. Raises: ValueError: If * `num_oov_buckets` is negative * `vocab_size` is not greater than zero * The `key_dtype` is not integer or string c ||fSr))vks rz*index_table_from_dataset..s !Qr)rT enumeratemap)rr@rPrQrArBrRs rindex_table_from_datasetr]s7\ G--/334GH+Z'D ::r)r:3tensorflow.python.data.experimental.ops.cardinalityrtensorflow.python.frameworkrrrtensorflow.python.opsrr-rr tensorflow.python.util.tf_exportr rTableInitializerBaser FastHashSpecrFrTr]rWrrrdsR.+.I,*6K6 12,88,3,^ 12#'("&%)#-#:#:!' T3Tn 78%)-.(,+-)3)@)@'-}}"& /:9/:r