AttributeError: 'Specificity' object has no attribute 'update_state_fn'

AttributeError: 'Specificity' object has no attribute 'update_state_fn' - python

I am trying to dump a model of MIL package and its metrics to the disk using pickle. I have no issues with the load function but when I try to extract the loaded contents (model, the metrics), I am getting an attribute error for the same.
I have added the code snippet for the metrics class here. You can also refer the trainer class from the author here
Is there any fix for this?
import numpy as np
import tensorflow as tf
from tensorflow.keras.backend import epsilon
from mil import metrics
class Metric(tf.keras.metrics.Metric):
""" Custom base class for implementing a metric,
each Metric subclass has to implement this methods. """
def __init__(self, name, **kwargs):
super(Metric, self).__init__(name=name, **kwargs)
def update_state_fn(self, y_true, y_pred, sample_weight=None):
""" Update the state of the metric
Parameters
----------
y_true : array-like containing the ground_truth
y_pred : array-like containing the network predictions
sample_weight : optional. weight more some predictions.
"""
raise NotImplementedError
def result(self):
""" Get the result of a metric """
raise NotImplementedError
def reset_states(self):
""" Reset the state of the metric """
raise NotImplementedError
class Specificity(Metric):
def __init__(self, name='specificity', **kwargs):
super(Specificity, self).__init__(name=name, **kwargs)
self.specificity = self.add_weight(name='specificity', initializer='zeros')
self.specificity.assign(np.nan)
self.tn = metrics.TrueNegatives()
self.fp = metrics.FalsePositives()
def update_state_fn(self, y_true, y_pred, sample_weight=None):
self.tn.update_state(y_true, y_pred, sample_weight)
self.fp.update_state(y_true, y_pred, sample_weight)
tn = self.tn.result()
fp = self.fp.result()
value = tf.where(tn + fp == 0, np.nan, tn / (tn + fp + epsilon()))
self.specificity.assign(value)
def result(self):
return self.specificity
def reset_states(self):
self.specificity.assign(np.nan)
self.tn.reset_states()
self.fp.reset_states()
I have added the entire code for calling this function: (from Github)
# importing dataset
from mil.data.datasets import musk1
# importing bag_representation
from mil.bag_representation import MILESMapping
# importing validation strategy
from mil.validators import LeaveOneOut
# importing final model, which in this case is the SVC classifier from sklearn
from mil.models import SVC
# importing trainer
from mil.trainer import Trainer
# importing preprocessing
from mil.preprocessing import StandarizerBagsList
# importing metrics, which in this case are from tf keras metrics
from mil.metrics import AUC
# loading dataset
(bags_train, y_train), (bags_test, y_test) = musk1.load()
# instantiate trainer
trainer = Trainer()
# preparing trainer
metrics = ['spec']
model = SVC(kernel='linear', C=1, class_weight='balanced')
pipeline = [('scale', StandarizerBagsList()), ('disc_mapping', MILESMapping())]
trainer.prepare(model, preprocess_pipeline=pipeline ,metrics=metrics)
# fitting trainer
valid = LeaveOneOut()
history = trainer.fit(bags_train, y_train, sample_weights='balanced', validation_strategy=valid, verbose=1)
# printing validation results for each fold
print(history['metrics_val'])
# predicting metrics for the test set
trainer.predict_metrics(bags_test, y_test)
import pickle
with open('model_pkl.pkl', 'wb') as files:
pickle.dump(trainer, files)
with open('/home/mylaptop/Res/model_pkl.pkl','rb') as sample_test:
abc = pickle.load(sample_test)
Stack trace:
AttributeError Traceback (most recent call last)
<ipython-input-84-9214b6d16ff4> in <module>
1 with open('/home/mylaptop/Res/model_pkl.pkl','rb') as sample_test:
----> 2 abc = pickle.load(sample_test)
AttributeError: 'Specificity' object has no attribute 'update_state_fn'

Related

Pytorch and ray tune: why the error; raise TuneError("Trials did not complete", incomplete_trials)?

I want to embed hyperparameter optimisation with ray into my pytorch script.
I wrote this code (which is a reproducible example):
## Standard libraries
CHECKPOINT_PATH = "/home/ad1/new_dev_v1"
DATASET_PATH = "/home/ad1/"
import torch
device = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu")
from importlib import reload
from itertools import *
import matplotlib
from itertools import groupby
from libs_public.api import get_pantry_token
from matplotlib import pyplot as plt
from matplotlib.colors import to_rgb
from openbabel import pybel
from openbabel.pybel import readstring,descs
from operator import itemgetter
from pathlib import Path
from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint
from pytorch_lightning.loggers import TensorBoardLogger
from ray import tune
from ray.tune import CLIReporter
from ray.tune.integration.pytorch_lightning import TuneReportCallback, TuneReportCheckpointCallback
from ray.tune.schedulers import ASHAScheduler, PopulationBasedTraining
from sklearn import preprocessing
from sklearn.metrics import f1_score, precision_score, recall_score,roc_auc_score
from socket import TIPC_DEST_DROPPABLE
from torch.nn import Linear
from torch.utils.data import TensorDataset
from torch_geometric.data import Data, Dataset,DataLoader,DenseDataLoader,InMemoryDataset
from torch_geometric.datasets import TUDataset
from torch_geometric.nn import GCNConv
from torch_geometric.nn import global_mean_pool
from torchmetrics.functional import precision_recall
from torchvision import transforms
from torchvision.datasets import CIFAR10
from tqdm.notebook import tqdm
import getpass, argparse
import joblib
import json
import logging
import math
import matplotlib.pyplot as plt
import networkx as nx
import numpy as np
import openbabel
import os
import pandas as pd
import pytorch_lightning as pl
import random
import re
import requests
import seaborn as sns
import sklearn
import sys
import time
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data as data
import torch_geometric
import torch_geometric.data as geom_data
import torch_geometric.nn as geom_nn
import torchmetrics
import torchvision
import warnings
matplotlib.rcParams['lines.linewidth'] = 2.0
pl.seed_everything(42)
print(device)
sns.reset_orig()
sns.set()
sys.path.append('/home/ad1/git/')
torch.backends.cudnn.deterministic = True
warnings.filterwarnings('ignore')
# Setting the seed
pl.seed_everything(42)
# Ensure that all operations are deterministic on GPU (if used) for reproducibility
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu")
print(device)
import torch
from torch_geometric.datasets import TUDataset
from torch.nn import Linear
from torch_geometric.nn import global_mean_pool
from torch_geometric.data import Data, Dataset,DataLoader
from torch.utils.data import TensorDataset
from ray import tune
from ray.tune import CLIReporter
from ray.tune.schedulers import ASHAScheduler, PopulationBasedTraining
from ray.tune.integration.pytorch_lightning import TuneReportCallback, TuneReportCheckpointCallback
dataset = TUDataset(root='/tmp/MUTAG', name='MUTAG', use_node_attr=True)
loader = DataLoader(dataset, batch_size=32, shuffle=True)
train_dataset = dataset
val_dataset = dataset
test_dataset = dataset
graph_train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
graph_val_loader = DataLoader(val_dataset, batch_size=64) # Additional loader if you want to change to a larger dataset
graph_test_loader = DataLoader(test_dataset, batch_size=64)
#will change this when it makes sense
#config = {
# "dropout": tune.uniform(0.4,0.5)
# }
config = {'dropout':0.4}
gnn_layer_by_name = {
"GCN": geom_nn.GCNConv,
"GAT": geom_nn.GATConv,
"GraphConv": geom_nn.GraphConv
}
class GCNLayer(nn.Module):
def __init__(self, c_in, c_out):
super().__init__()
self.projection = nn.Linear(c_in, c_out)
def forward(self, node_feats, adj_matrix):
"""
Inputs:
node_feats - Tensor with node features of shape [batch_size, num_nodes, c_in]
adj_matrix - Batch of adjacency matrices of the graph. If there is an edge from i to j, adj_matrix[b,i,j]=1 else 0.
Supports directed edges by non-symmetric matrices. Assumes to already have added the identity connections.
Shape: [batch_size, num_nodes, num_nodes]
"""
# Num neighbours = number of incoming edges
num_neighbours = adj_matrix.sum(dim=-1, keepdims=True)
node_feats = self.projection(node_feats)
node_feats = torch.bmm(adj_matrix, node_feats)
node_feats = node_feats / num_neighbours
return node_feats
class GNNModel(nn.Module):
def __init__(self, c_in, c_hidden, c_out, num_layers=2, layer_name="GCN", dp_rate=config['dropout'], **kwargs):
"""
Inputs:
c_in - Dimension of input features
c_hidden - Dimension of hidden features
c_out - Dimension of the output features. Usually number of classes in classification
num_layers - Number of "hidden" graph layers
layer_name - String of the graph layer to use
dp_rate - Dropout rate to apply throughout the network
kwargs - Additional arguments for the graph layer (e.g. number of heads for GAT)
"""
super().__init__()
gnn_layer = gnn_layer_by_name[layer_name]
layers = []
in_channels, out_channels = c_in, c_hidden
for l_idx in range(num_layers-1):
layers += [
gnn_layer(in_channels=in_channels,
out_channels=out_channels,
**kwargs),
nn.ReLU(inplace=True),
nn.Dropout(config['dropout'])
]
in_channels = c_hidden
layers += [gnn_layer(in_channels=in_channels,
out_channels=c_out,
**kwargs)]
self.layers = nn.ModuleList(layers)
def forward(self, x, edge_index):
"""
Inputs:
x - Input features per node
edge_index - List of vertex index pairs representing the edges in the graph (PyTorch geometric notation)
"""
for l in self.layers:
# For graph layers, we need to add the "edge_index" tensor as additional input
# All PyTorch Geometric graph layer inherit the class "MessagePassing", hence
# we can simply check the class type.
if isinstance(l, geom_nn.MessagePassing):
x = l(x, edge_index)
else:
x = l(x)
return x
class GraphGNNModel(nn.Module):
def __init__(self, c_in, c_hidden, c_out, dp_rate_linear=0.5, **kwargs):
"""
Inputs:
c_in - Dimension of input features
c_hidden - Dimension of hidden features
c_out - Dimension of output features (usually number of classes)
dp_rate_linear - Dropout rate before the linear layer (usually much higher than inside the GNN)
kwargs - Additional arguments for the GNNModel object
"""
super().__init__()
self.GNN = GNNModel(c_in=c_in,
c_hidden=c_hidden,
c_out=c_hidden, # Not our prediction output yet!
**kwargs)
self.head = nn.Sequential(
nn.Dropout(config['dropout']),
nn.Linear(c_hidden, c_out)
)
def forward(self, x, edge_index, batch_idx):
"""
Inputs:
x - Input features per node
edge_index - List of vertex index pairs representing the edges in the graph (PyTorch geometric notation)
batch_idx - Index of batch element for each node
"""
x = self.GNN(x, edge_index)
x = geom_nn.global_mean_pool(x, batch_idx) # Average pooling
x = self.head(x)
return x
#see https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html
class GraphLevelGNN(pl.LightningModule):
def __init__(self, **model_kwargs):
super().__init__()
# Saving hyperparameters
self.save_hyperparameters()
self.model = GraphGNNModel(**model_kwargs)
self.loss_module = nn.BCEWithLogitsLoss() #if self.hparams.c_out == 1 else nn.CrossEntropyLoss()
def forward(self, data, mode="train"):
x, edge_index, batch_idx = data.x, data.edge_index, data.batch
x = self.model(x, edge_index, batch_idx)
x = x.squeeze(dim=-1)
if self.hparams.c_out == 1:
preds = (x > 0).float()
data.y = data.y.float()
else:
preds = x.argmax(dim=-1)
loss = self.loss_module(x, data.y)
acc = (preds == data.y).sum().float() / preds.shape[0]
f1 = f1_score(preds,data.y) ##change f1/precision and recall was just testing
precision = precision_score(preds,data.y)
recall = recall_score(preds,data.y)
#roc_auc = roc_auc_score(preds,data.y) ##ADD THIS BACK IN
return loss, acc, f1,precision, recall
def configure_optimizers(self):
optimizer = optim.SGD(self.parameters(),lr=0.1) # High lr because of small dataset and small model
return optimizer
def training_step(self, batch, batch_idx):
loss, acc, _,_, _ = self.forward(batch, mode="train")
self.log('train_loss', loss,on_epoch=True,logger=True)
self.log('train_acc', acc,on_epoch=True,logger=True)
#self.log('train_precision',precision_and_recall)
return loss
def validation_step(self, batch, batch_idx):
loss, acc, _,_, _ = self.forward(batch, mode="val")
self.log('val_acc', acc,on_epoch=True,logger=True)
self.log('val_loss', loss,on_epoch=True,logger=True)
def test_step(self, batch, batch_idx):
loss, acc, f1,precision, recall = self.forward(batch, mode="test")
self.log('test_acc', acc,on_epoch=True,logger=True)
self.log('test_f1', f1,on_epoch=True,logger=True)
self.log('test_precision', precision,on_epoch=True,logger=True)
self.log('test_recall', recall,on_epoch=True,logger=True)
#self.log('roc_auc', roc_auc,on_epoch=True,logger=True)
from pytorch_lightning import loggers as pl_loggers
def train_graph_classifier(model_name, **model_kwargs):
pl.seed_everything(42)
# Create a PyTorch Lightning trainer with the generation callback
root_dir = os.path.join(CHECKPOINT_PATH, "GraphLevel" + model_name)
os.makedirs(root_dir, exist_ok=True)
csv_logger = pl_loggers.CSVLogger(save_dir="logs/")
tune_report_callback = TuneReportCheckpointCallback(
metrics={
"val_loss": "val_loss",
"val_acc": "val_acc",
},
filename="ray_ckpt",
on="validation_end",
)
trainer = pl.Trainer(default_root_dir=root_dir,
callbacks=[ModelCheckpoint(save_weights_only=True, mode="max", monitor="val_acc"),tune_report_callback],
# TuneReportCallback(
# {
# "loss": "val_loss",
# "mean_accuracy": "val_accuracy"
# },
# on="test_end")] # need to change this to validation but error at the minute
# ,
gpus=1 if str(device).startswith("cuda") else 0,
max_epochs=3,
progress_bar_refresh_rate=1,
logger=csv_logger,
)
trainer.logger._default_hp_metric = None # Optional logging argument that we don't need
# Check whether pretrained model exists. If yes, load it and skip training
pretrained_filename = os.path.join(CHECKPOINT_PATH, f"GraphLevel{model_name}.ckpt")
if os.path.isfile(pretrained_filename):
print("Found pretrained model, loading...")
model = GraphLevelGNN.load_from_checkpoint(pretrained_filename)
else:
pl.seed_everything(42)
model = GraphLevelGNN(c_in = dataset.num_node_features,
c_out=1, #if tu_dataset.num_classes==2 else tu_dataset.num_classes,
**model_kwargs)
trainer.fit(model, graph_train_loader, graph_val_loader)
model = GraphLevelGNN.load_from_checkpoint(trainer.checkpoint_callback.best_model_path)
# Test best model on validation and test set
#train_result = trainer.test(model, graph_train_loader, verbose=False)
#test_result = trainer.test(model, graph_test_loader, verbose=False)
#result = {"test": test_result[0]['test_acc'], "train": train_result[0]['test_acc']}
#return model, result
return model
# Example of ASHA Scheduler
scheduler_asha = ASHAScheduler(
max_t=100,
grace_period=1,
reduction_factor=2,
)
from ray.tune.integration.pytorch_lightning import TuneReportCallback, TuneReportCheckpointCallback
reporter = CLIReporter(
parameter_columns=['dropout'],
metric_columns=["val_loss", "val_acc", "training_iteration"]
)
model = train_graph_classifier(model_name="GraphConv",
c_hidden=128,
layer_name="GraphConv",
num_layers=3,
dp_rate_linear=0.5,
dp_rate=0.0)
result = tune.run(
tune.with_parameters(
model,
#feature_size=10,
#target_size=2,
epochs=50,
gpus=0
),
resources_per_trial={
"cpu": 1,
"gpu": 0,
},
local_dir='/home/ad1/ray_ckpt2', # path for saving checkpoints
metric="val_loss",
mode="min",
config=config,
num_samples=16,
scheduler=scheduler_asha,
progress_reporter=reporter,
name="test",
)
And the error returned is:
(tune_with_parameters pid=65319) 2022-08-17 16:28:47,053 ERROR function_runner.py:286 -- Runner Thread raised error.
(tune_with_parameters pid=65319) Traceback (most recent call last):
(tune_with_parameters pid=65319) File "/root/miniconda3/lib/python3.7/site-packages/ray/tune/function_runner.py", line 277, in run
(tune_with_parameters pid=65319) self._entrypoint()
(tune_with_parameters pid=65319) File "/root/miniconda3/lib/python3.7/site-packages/ray/tune/function_runner.py", line 352, in entrypoint
(tune_with_parameters pid=65319) self._status_reporter.get_checkpoint(),
(tune_with_parameters pid=65319) File "/root/miniconda3/lib/python3.7/site-packages/ray/util/tracing/tracing_helper.py", line 462, in _resume_span
(tune_with_parameters pid=65319) return method(self, *_args, **_kwargs)
(tune_with_parameters pid=65319) File "/root/miniconda3/lib/python3.7/site-packages/ray/tune/function_runner.py", line 645, in _trainable_func
(tune_with_parameters pid=65319) output = fn()
(tune_with_parameters pid=65319) File "/root/miniconda3/lib/python3.7/site-packages/ray/tune/utils/trainable.py", line 410, in inner
(tune_with_parameters pid=65319) trainable(config, **fn_kwargs)
(tune_with_parameters pid=65319) File "/root/miniconda3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
(tune_with_parameters pid=65319) return forward_call(*input, **kwargs)
(tune_with_parameters pid=65319) TypeError: forward() got an unexpected keyword argument 'checkpoint_dir'
Traceback (most recent call last):
File "test_pytorch.py", line 390, in <module>
name="test",
File "/root/miniconda3/lib/python3.7/site-packages/ray/tune/tune.py", line 741, in run
raise TuneError("Trials did not complete", incomplete_trials)
ray.tune.error.TuneError: ('Trials did not complete', [tune_with_parameters_a90c2_00000, tune_with_parameters_a90c2_00001,
...cut for space
tune_with_parameters_a90c2_00014, tune_with_parameters_a90c2_00015])
Could someone show me where I'm going wrong, how to I run HPO with tune in this network and then train the model with the best hyperparameters and then return the model for prediction?

Ray Tune expects a function trainable in the form of
def train_fn(config):
# ...
In your case, it is probably best to wrap the train_graph_classifier function, e.g.
def train_fn(config):
train_graph_classifier(
model_name="GraphConv",
layer_name="GraphConv",
**config)
analysis = tune.run(
train_fn,
config={
# provide your hyperparameter search space here
"c_hidden": tune.choice([64, 128]),
"dp_rate_linear": tune.quniform(0.0, 1.0, 0.1),
# ...
},
metric="val_loss",
mode="min",
# ...
print(analysis.best_checkpoint)
If you provide the TuneReportCheckpointCallback to the trainer, the analysis.best_checkpoint should contain the best model that can be then loaded for prediction, e.g.
with analysis.best_checkpoint.as_directory() as tmpdir:
trainer = GraphLevelGNN.load_from_checkpoint(tmpdir)

Cannot add CRF layer on top of BERT in keras for NER

I am facing an unknown issue while training my BERT-CRF model for NER. I am using keras.contrib for the CRF model.
Here are the imported libraries.
!pip install transformers
!pip install git+https://www.github.com/keras-team/keras-contrib.git
import pandas as pd
import numpy as np
from transformers import TFBertModel, BertTokenizer, BertConfig
import tensorflow as tf
from tensorflow import keras
from keras_contrib.layers import CRF
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from tqdm import tqdm
Code for the model creation.
input_ids = keras.layers.Input(shape=(MAX_LEN,), dtype=tf.int32)
token_type_ids = keras.layers.Input(shape=(MAX_LEN,), dtype=tf.int32)
attention_mask = keras.layers.Input(shape=(MAX_LEN,), dtype=tf.int32)
bert_output = bert(
[input_ids,
attention_mask,
token_type_ids]
)[0]
bert_output = keras.layers.Dropout(0.3)(bert_output)
dense_layer_output = keras.layers.Dense(num_classes+1, activation='softmax', name='output')(bert_output)
crf = CRF(num_classes)
outputs = crf(dense_layer_output)
model = keras.Model(
inputs=[input_ids, token_type_ids, attention_mask],
outputs=[outputs],
)
model.compile(
loss=crf.loss_function,
metrics=[crf.accuracy],
optimizer=keras.optimizers.Adam(5e-5)
)
model.fit(
x_train,
y_train,
epochs=1,
verbose=1,
batch_size=32,
validation_data=(x_test, y_test)
)
While trying to train the model I am getting this error. I cannot understand from where it is originating and why.
WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-18-f369b38eb91d> in <module>()
5 verbose=1,
6 batch_size=32,
----> 7 validation_data=(x_test, y_test)
8 )
9 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
975 except Exception as e: # pylint:disable=broad-except
976 if hasattr(e, "ag_error_metadata"):
--> 977 raise e.ag_error_metadata.to_exception(e)
978 else:
979 raise
AttributeError: in user code:
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:805 train_function *
return step_function(self, iterator)
/usr/local/lib/python3.7/dist-packages/keras_contrib/losses/crf_losses.py:54 crf_loss *
crf, idx = y_pred._keras_history[:2]
AttributeError: 'Tensor' object has no attribute '_keras_history'
I have read on the internet that keras.contrib is depricated but I don't know any other way how to use a CRF layer on top of BERT. If there is a better way of doing it in keras then please suggest me.
I don't know whether this question is making sense or not but any help would be appreciated.
Thanks in advance!

The easiest way is to use the CRF layer of the TensorFlow addons. Then utilize the output of that to calculate the loss.
import tensorflow_addons as tfa
crf = tfa.layers.CRF(len(num_labels)+1)
Further, you can utilize it by creating your own Model class too for model creation.
from tensorflow_addons.text.crf import crf_log_likelihood
def unpack_data(data):
if len(data) == 2:
return data[0], data[1], None
elif len(data) == 3:
return data
else:
raise TypeError("Expected data to be a tuple of size 2 or 3.")
class ModelWithCRFLoss(tf.keras.Model):
"""Wrapper around the base model for custom training logic."""
def __init__(self, base_model):
super().__init__()
self.base_model = base_model
def call(self, inputs):
return self.base_model(inputs)
def compute_loss(self, x, y, sample_weight, training=False):
y_pred = self(x, training=training)
_, potentials, sequence_length, chain_kernel = y_pred
# we now add the CRF loss:
crf_loss = -crf_log_likelihood(potentials, y, sequence_length, chain_kernel)[0]
if sample_weight is not None:
crf_loss = crf_loss * sample_weight
return tf.reduce_mean(crf_loss), sum(self.losses)
def train_step(self, data):
x, y, sample_weight = unpack_data(data)
with tf.GradientTape() as tape:
crf_loss, internal_losses = self.compute_loss(
x, y, sample_weight, training=True
)
total_loss = crf_loss + internal_losses
gradients = tape.gradient(total_loss, self.trainable_variables)
self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))
return {"crf_loss": crf_loss, "internal_losses": internal_losses}
def test_step(self, data):
x, y, sample_weight = unpack_data(data)
crf_loss, internal_losses = self.compute_loss(x, y, sample_weight)
return {"crf_loss_val": crf_loss, "internal_losses_val": internal_losses}
You can write along these lines of code
decoded_sequence, potentials, sequence_length, chain_kernel = crf(dense_layer_output, mask=attention_mask)
base_model = tf.keras.Model(
inputs=[input_ids, attention_mask],
outputs=crf_layer_outputs,
)
model = ModelWithCRFLoss(base_model)
model.compile(
optimizer=tf.keras.optimizers.Adam(learning_rate=5e-3, epsilon=1e-08),
metrics=tf.metrics.SparseCategoricalAccuracy(),
)

Getting feature names from a pipeline with tfidfvectorizer

I have been trying to get the feature names on my model for quite some time now but have a hard time understanding how to do it. I have tried many posts on here but can't get it to work. Here is my code:
loading the classes I need to combine tfidfvectorizer with other features
from sklearn.base import TransformerMixin, BaseEstimator
class ItemSelector(BaseEstimator, TransformerMixin):
def __init__(self, key):
self.key = key
def fit(self, x, y=None):
return self
def transform(self, data_dict):
return data_dict[self.key]
class FeatureTypeSelector(TransformerMixin, BaseEstimator):
FEATURE_TYPES = {
'categorical': [
'COLUMN_A','COLUMN_B'
],
'continuous': [
'COLULMN_C','COLUMN_D'
]
}
def __init__(self, feature_type):
self.columns = self.FEATURE_TYPES[feature_type]
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.columns]
class RowToDictTransformer(TransformerMixin, BaseEstimator):
def fit(self, X, y=None):
return self
def transform(self, X):
return (row[1] for row in X.iterrows())
Then the code to put everything in a pipeline and run the regressor
from sklearn.pipeline import make_union, make_pipeline
from sklearn.feature_extraction import DictVectorizer
from sklearn.preprocessing import RobustScaler
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import MinMaxScaler
# Create the preprocessor
preprocessor = make_union(
make_pipeline(
ItemSelector(key='TEXT_COLUMN'),
TfidfVectorizer(lowercase=False, min_df=1),
),
make_pipeline(
FeatureTypeSelector('continuous'),
MinMaxScaler(),
),
make_pipeline(
FeatureTypeSelector('categorical'),
RowToDictTransformer(),
DictVectorizer(sparse=False), # set sparse=True if you get MemoryError
),
)
# fit and transform the data
preprocessor.fit_transform(x_train)
# choose some estimator
# estimator = MultinomialNB()
estimator = LinearRegression()
# Create the model
model = make_pipeline(preprocessor, estimator)
# Training the model
model.fit(x_train, y_train)
# Predicting the model
predicted = model.predict(x_test)
I can run the model.coef_ to get all the coefficients but I want to see how each item of the TEXT_COLUMN is affected by which weight. I have tried calling get_feature_names() or tried passing them in the pipeline but with no succes (most of google's results are purple by now).
Anyone that can give me a bit of guidance how to pass the feature names to the end of the pipeline? The ideal result would be a dataframe with the feature (row from the TEXT_COLUMN) and feature_weight as value.

How to set keras custom metrics to only be called at the end of epoch?

I am trying to use a custom metrics for my neural network and this metric should only be evaluated at the end of the epoch. The problem that I encounter is that the metrics is evaluated at each batch which is not the behaviour wanted. Note that I am working with generators and fit_generator with keras.
validation_data are loaded with a generator that implements keras.utils.Sequence
class DataGenerator(keras.utils.Sequence):
def __init__(self, inputs, labels, batch_size):
self.inputs = inputs
self.labels = labels
self.batch_size = batch_size
def __getitem__(self, index):
#some processing done here
return batch_inputs, batch_labels
def __len__(self):
return int(np.floor(len(self.inputs) / self.batch_size))
I tried to implement what the keras documentation suggests but I did not find any info to specify the metric should only used at the end of epoch.
def auc_roc(y_true, y_pred):
auc, up_opt = tf.metrics.auc(y_true, y_pred)
K.get_session().run(tf.local_variables_initializer())
with tf.control_dependencies([up_opt]):
auc = tf.identity(auc)
return auc
So right now the auc_roc is called after each batch instead of a single call at the end of the epoch.

from sklearn.metrics import roc_auc_score
from keras.callbacks import Callback
class IntervalEvaluation(Callback):
def __init__(self, validation_data=(), interval=10):
super(Callback, self).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict_proba(self.X_val, verbose=0)
score = roc_auc_score(self.y_val, y_pred)
print("interval evaluation - epoch: {:d} - score: {:.6f}".format(epoch, score))
Usage:
ival = IntervalEvaluation(validation_data=(x_test2, y_test2), interval=1)
More Info:
http://digital-thinking.de/keras-three-ways-to-use-custom-validation-metrics-in-keras/

error when using keras' sk-learn API

  i'm learning keras these days, and i met an error when using scikit-learn API.Here are something maybe useful:
ENVIRONMENT:
python:3.5.2
keras:1.0.5
scikit-learn:0.17.1
CODE
import pandas as pd
from keras.layers import Input, Dense
from keras.models import Model
from keras.models import Sequential
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.cross_validation import train_test_split
from sklearn.cross_validation import cross_val_score
from sqlalchemy import create_engine
from sklearn.cross_validation import KFold
def read_db():
"get prepared data from mysql."
con_str = "mysql+mysqldb://root:0000#localhost/nbse?charset=utf8"
engine = create_engine(con_str)
data = pd.read_sql_table('data_ml', engine)
return data
def nn_model():
"create a model."
model = Sequential()
model.add(Dense(output_dim=100, input_dim=105, activation='softplus'))
model.add(Dense(output_dim=1, input_dim=100, activation='softplus'))
model.compile(loss='mean_squared_error', optimizer='adam')
return model
data = read_db()
y = data.pop('PRICE').as_matrix()
x = data.as_matrix()
model = nn_model()
model = KerasRegressor(build_fn=model, nb_epoch=2)
model.fit(x,y) #something wrong here!
ERROR
Traceback (most recent call last):
File "C:/Users/Administrator/PycharmProjects/forecast/gridsearch.py", line 43, in <module>
model.fit(x,y)
File "D:\Program Files\Python35\lib\site-packages\keras\wrappers\scikit_learn.py", line 135, in fit
**self.filter_sk_params(self.build_fn.__call__))
TypeError: __call__() missing 1 required positional argument: 'x'
Process finished with exit code 1
  the model works well without packaging with kerasRegressor, but i wanna using sk_learn's gridSearch after this, so i'm here for help. I tried but still have no idea.
something maybe helpful:
keras.warappers.scikit_learn.py
class BaseWrapper(object):
def __init__(self, build_fn=None, **sk_params):
self.build_fn = build_fn
self.sk_params = sk_params
self.check_params(sk_params)
def fit(self, X, y, **kwargs):
'''Construct a new model with build_fn and fit the model according
to the given training data.
# Arguments
X : array-like, shape `(n_samples, n_features)`
Training samples where n_samples in the number of samples
and n_features is the number of features.
y : array-like, shape `(n_samples,)` or `(n_samples, n_outputs)`
True labels for X.
kwargs: dictionary arguments
Legal arguments are the arguments of `Sequential.fit`
# Returns
history : object
details about the training history at each epoch.
'''
if self.build_fn is None:
self.model = self.__call__(**self.filter_sk_params(self.__call__))
elif not isinstance(self.build_fn, types.FunctionType):
self.model = self.build_fn(
**self.filter_sk_params(self.build_fn.__call__))
else:
self.model = self.build_fn(**self.filter_sk_params(self.build_fn))
loss_name = self.model.loss
if hasattr(loss_name, '__name__'):
loss_name = loss_name.__name__
if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
y = to_categorical(y)
fit_args = copy.deepcopy(self.filter_sk_params(Sequential.fit))
fit_args.update(kwargs)
history = self.model.fit(X, y, **fit_args)
return history
  error occored in this line:
self.model = self.build_fn(
**self.filter_sk_params(self.build_fn.__call__))
self.build_fn here is keras.models.Sequential
models.py
class Sequential(Model):
def call(self, x, mask=None):
if not self.built:
self.build()
return self.model.call(x, mask)
So, what's that x mean and how to fix this error?
Thanks!

xiao, I ran into the same issue! Hopefully this helps:
Background and The Issue
The documentation for Keras states that, when implementing Wrappers for scikit-learn, there are two arguments. The first is the build function, which is a "callable function or class instance". Specifically, it states that:
build_fn should construct, compile and return a Keras model, which will then be used to fit/predict. One of the following three values could be passed to build_fn:
A function
An instance of a class that implements the call method
None. This means you implement a class that inherits from either KerasClassifier or KerasRegressor. The call method of the present class will then be treated as the default build_fn.
In your code, you create the model, and then pass the model as the value for the argument build_fn when creating the KerasRegressor wrapper:
model = nn_model()
model = KerasRegressor(build_fn=model, nb_epoch=2)
Herein lies the issue. Rather than passing your nn_model function as the build_fn, you pass an actual instance of the Keras Sequential model. For this reason, when fit() is called, it cannot find the call method, because it is not implemented in the class you returned.
Proposed Solution
What I did to make things work is pass the function as build_fn, rather than an actual model:
data = read_db()
y = data.pop('PRICE').as_matrix()
x = data.as_matrix()
# model = nn_model() # Don't do this!
# set build_fn equal to the nn_model function
model = KerasRegressor(build_fn=nn_model, nb_epoch=2) # note that you do not call the function!
model.fit(x,y) # fixed!
This is not the only solution (you could set build_fn to a class that implements the call method appropriately), but the one that worked for me. I hope it helps you!

User-defined keyword arguments passed to __init__() that is to say, all keyword arguments that were given to __init__() will be passed to model_build_fn directly. For example, calling KerasClassifier(myparam=10) will result in a model_build_fn(my_param=10)
here's an example:
class MyMultiOutputKerasRegressor(KerasRegressor):
# initializing
def __init__(self, **kwargs):
KerasRegressor.__init__(self, **kwargs)
# simpler fit method
def fit(self, X, y, **kwargs):
KerasRegressor.fit(self, X, [y]*3, **kwargs)
(...)
def get_quantile_reg_rpf_nn(layers_shape=[50,100,200,100,50], inDim= 4, outDim=1, act='relu'):
# do model stuff...
(...)
initialize the Keras regressor:
base_model = MyMultiOutputKerasRegressor(build_fn=get_quantile_reg_rpf_nn,
layers_shape=[50,100,200,100,50], inDim= 4,
outDim=1, act='relu', epochs=numEpochs,
batch_size=batch_size, verbose=0)

Develop Reference

Python is a programming language that lets you work quickly and integrate systems more effectively.

AttributeError: 'Specificity' object has no attribute 'update_state_fn' - python

Related

Pytorch and ray tune: why the error; raise TuneError("Trials did not complete", incomplete_trials)?

Cannot add CRF layer on top of BERT in keras for NER

Getting feature names from a pipeline with tfidfvectorizer

How to set keras custom metrics to only be called at the end of epoch?

error when using keras' sk-learn API

Categories

Resources