Develop Reference

How to pass multiple value in gitlab cicd variables

I have a multiple project in my gitlab repository wherein I do perform multiple commits when it requires.
I have develop a code in python through which I can get report of all the commits done by me in a csv format for all the projects available in gitlab repository as I have hard coded the the project ids in my python code as a LIST.
The Header of the csv file is : Date, submitted, gitlab_url, project, username, subject.
Now I want to run the pipeline manually by setting up an environment variable as 'Project_Ids'
and want to pass some of the project ids as value (More than one project id as a value) so that csv report should get generated for only these projects which has been passed as a value in environment variable.
so My question is , How can I pass multiple project ids as a value in 'Project_Ids' key while running the pipeline manually.
enter image description here
import gitlab
import os
import datetime
import csv
import re
Project_id_list = ['9427','8401','17937','26813','24899','23729','34779','27638','28600']
headerList = ['Date', 'Submitted', 'Gitlab_url', 'Project', 'Branch', 'Status', 'Username', 'Ticket', 'Subject']
filename = 'mydemo_{}'.format(datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S'))
# private token authentication
gl = gitlab.Gitlab('https://main.gitlab.in.com/', private_token="MLyWwLyEhU2zZjjjhZXog")
gl.auth()
# list all projects
for m in Project_id_list:
i=0
if (i<len(Project_id_list)):
i=+1
print(m)
projects = gl.projects.get(m)
commits = projects.commits.list(all=True, query_parameters={'ref_name': 'master'})
with open(f"{filename}_{m}.csv", 'w', newline="") as file:
dw = csv.DictWriter(file, delimiter=',',
fieldnames=headerList)
dw.writeheader()
for commit in commits:
print(commit)
msg = commit.message
if 'master' in msg or 'LCS-' in msg:
projectName = projects.path_with_namespace
branch = 'master'
status = 'merged'
date = commit.committed_date.split('T')[0]
submitted1 = commit.created_at.split('T')[1]
submitted = submitted1.split('.000')[0]
Gitlab_url = commit.web_url.split('-')[0]
username = commit.author_name
subject = commit.title
subject1 = commit.message.splitlines()
print(subject1)
subject2 = subject1[0:3]
print(subject2)
subject3 = ' '.join(subject2)
print(subject3)
match = re.search('S-\d+', subject3)
if match:
ticket = match.group(0)
ticket_url = 'https://.in.com/browse/' + str(ticket)
ticket1 = ticket_url
dw.writerow({'Date': date, 'Submitted': submitted, 'Gitlab_url': Gitlab_url, 'Project': projectName,
'Branch': branch, 'Status': status, 'Username': username, 'Ticket': ticket1,
'Subject': subject3})
else:
ticket1 = 'Not Found'
dw.writerow({'Date': date, 'Submitted': submitted, 'Gitlab_url': Gitlab_url, 'Project': projectName,
'Branch': branch, 'Status': status, 'Username': username, 'Ticket': ticket1,
'Subject': subject3})

Just use a space or some other delimiter in the variable value. For example, a string like 123 456 789
Then in Python, simply parse the variable. For example, using the string .split method to split on whitespace.
import os
...
project_ids_variable = os.environ.get('PROJECT_IDS', '') # '123 456 789'
project_ids = project_ids_variable.split() # ['123', '456', '789']
for project_id in project_ids:
project = gl.projects.get(project_id)
print(project)

Huggingface SciBERT predict masked word not working

I am trying to use the pretrained SciBERT model (https://huggingface.co/allenai/scibert_scivocab_uncased) from Huggingface to predict masked words in scientific/biomedical text. This produces errors, and not sure how to move forward from this point.
Here is the code so far -
!pip install transformers
from transformers import pipeline, AutoTokenizer, AutoModel
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased")
model = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased")
unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer)
unmasker("the patient is a 55 year old [MASK] admitted with pneumonia")
This works with BERT alone, but is not the specialized pre-trained model -
!pip install transformers
from transformers import pipeline
unmasker = pipeline('fill-mask', model='bert-base-uncased')
unmasker("the patient is a 55 year old [MASK] admitted with pneumonia")
The errors with SciBERT are -
/usr/local/lib/python3.7/dist-packages/transformers/pipelines/__init__.py in pipeline(task, model, config, tokenizer, feature_extractor, framework, revision, use_fast, use_auth_token, model_kwargs, **kwargs)
494 kwargs["feature_extractor"] = feature_extractor
495
--> 496 return task_class(model=model, framework=framework, task=task, **kwargs)
/usr/local/lib/python3.7/dist-packages/transformers/pipelines/fill_mask.py in __init__(self, model, tokenizer, modelcard, framework, args_parser, device, top_k, task)
73 )
74
---> 75 self.check_model_type(TF_MODEL_WITH_LM_HEAD_MAPPING if self.framework == "tf" else MODEL_FOR_MASKED_LM_MAPPING)
76 self.top_k = top_k
77
/usr/local/lib/python3.7/dist-packages/transformers/pipelines/base.py in check_model_type(self, supported_models)
652 self.task,
653 self.model.base_model_prefix,
--> 654 f"The model '{self.model.__class__.__name__}' is not supported for {self.task}. Supported models are {supported_models}",
655 )
656
PipelineException: The model 'BertModel' is not supported for fill-mask. Supported models are ['BigBirdForMaskedLM', 'Wav2Vec2ForMaskedLM', 'ConvBertForMaskedLM', 'LayoutLMForMaskedLM', 'DistilBertForMaskedLM', 'AlbertForMaskedLM', 'BartForConditionalGeneration', 'MBartForConditionalGeneration', 'CamembertForMaskedLM', 'XLMRobertaForMaskedLM', 'LongformerForMaskedLM', 'RobertaForMaskedLM', 'SqueezeBertForMaskedLM', 'BertForMaskedLM', 'MegatronBertForMaskedLM', 'MobileBertForMaskedLM', 'FlaubertWithLMHeadModel', 'XLMWithLMHeadModel', 'ElectraForMaskedLM', 'ReformerForMaskedLM', 'FunnelForMaskedLM', 'MPNetForMaskedLM', 'TapasForMaskedLM', 'DebertaForMaskedLM', 'DebertaV2ForMaskedLM', 'IBertForMaskedLM']

As the error message tells you, you need to use AutoModelForMaskedLM:
from transformers import pipeline, AutoTokenizer, AutoModelForMaskedLM
tokenizer = AutoTokenizer.from_pretrained("allenai/scibert_scivocab_uncased")
model = AutoModelForMaskedLM.from_pretrained("allenai/scibert_scivocab_uncased")
unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer)
unmasker("the patient is a 55 year old [MASK] admitted with pneumonia")
Output:
[{'sequence': 'the patient is a 55 year old woman admitted with pneumonia',
'score': 0.4025486707687378,
'token': 10221,
'token_str': 'woman'},
{'sequence': 'the patient is a 55 year old man admitted with pneumonia',
'score': 0.23970800638198853,
'token': 508,
'token_str': 'man'},
{'sequence': 'the patient is a 55 year old female admitted with pneumonia',
'score': 0.15444642305374146,
'token': 3672,
'token_str': 'female'},
{'sequence': 'the patient is a 55 year old male admitted with pneumonia',
'score': 0.1111455038189888,
'token': 3398,
'token_str': 'male'},
{'sequence': 'the patient is a 55 year old boy admitted with pneumonia',
'score': 0.015877680853009224,
'token': 12481,
'token_str': 'boy'}]

Odoo - Confirm Sales Order

Hi I am using Odoo10 and trying to create a sales order in POS, below code creates the Sales orders quotation. I want to Confirm the Sale and create Sales order not quotation.
#api.model
def create_sales_order(self, orderline, customer_id, sign):
sale_pool = self.env['sale.order']
prod_pool = self.env['product.product']
sale_line_pool = self.env['sale.order.line']
sale_no = ''
sale = {}
if customer_id:
customer_id = int(customer_id)
sale = {'partner_id': customer_id,
'partner_invoice_id': customer_id,
'partner_shipping_id': customer_id,
'signature': sign}
sale_id = sale_pool.create(sale)
if sale_id:
sale_brw = sale_id
sale_brw.onchange_partner_id()
#create sale order line
for line in orderline:
sale_line = {}
if line.get('product_id'):
prod_rec = prod_pool.browse(line['product_id'])
sale_line.update({'name': prod_rec.name or False,
'product_id': prod_rec.id,
'product_uom_qty': line['qty'],
'discount': line.get('discount'),
'order_id': sale_id.id})
sale_line_id = sale_line_pool.create(sale_line)
for line in sale_line_id:
line.product_id_change()
return {"name": sale_brw.name, "id": sale_brw.id }
How do I create sales order not quotation?

Short answer: Set state to "sale":
sale = {'partner_id': customer_id,
'partner_invoice_id': customer_id,
'partner_shipping_id': customer_id,
'signature': sign,
'state': 'sale'}
Sale orders and Quotations are saved on the same model (namely, sale.order) you can tell whether it is a SO or a Quote by looking at its state:
state | Meaning
-------|--------
draft | Quotation
sent | Quotation Sent
sale | Sales Order
done | Locked
cancel | Cancelled
Also, you can look at the function action_confirm which is triggered by clicking on the Confirm Sale button on addons/sale/models/sale.py file:
445 def action_done(self):
446 return self.write({'state': 'done'})
...
451 #api.multi
452 def action_confirm(self):
453 for order in self:
454 order.state = 'sale'
455 order.confirmation_date = fields.Datetime.now()
456 if self.env.context.get('send_email'):
457 self.force_quotation_send()
458 order.order_line._action_procurement_create()
459 if self.env['ir.values'].get_default('sale.config.settings', 'auto_done_setting'):
460 self.action_done()
461 return True

I use to_gbq on pandas for updating Google BigQuery and get GenericGBQException

While trying to use to_gbq for updating Google BigQuery table, I get a response of:
GenericGBQException: Reason: 400 Error while reading data, error message: JSON table encountered too many errors, giving up. Rows: 1; errors: 1.
My code:
gbq.to_gbq(mini_df,'Name-of-Table','Project-id',chunksize=10000,reauth=False,if_exists='append',private_key=None)
and my dataframe of mini_df looks like:
date request_number name feature_name value_name value
2018-01-10 1 1 "a" "b" 0.309457
2018-01-10 1 1 "c" "d" 0.273748
While I'm running the to_gbq, and there's no table on the BigQuery, I can see that the table is created with the next schema:
date STRING NULLABLE
request_number STRING NULLABLE
name STRING NULLABLE
feature_name STRING NULLABLE
value_name STRING NULLABLE
value FLOAT NULLABLE
What am I doing wrong? How can I solve this?
P.S, rest of the exception:
BadRequest Traceback (most recent call last)
~/anaconda3/envs/env/lib/python3.6/site-packages/pandas_gbq/gbq.py in load_data(self, dataframe, dataset_id, table_id, chunksize)
589 destination_table,
--> 590 job_config=job_config).result()
591 except self.http_error as ex:
~/anaconda3/envs/env/lib/python3.6/site-packages/google/cloud/bigquery/job.py in result(self, timeout)
527 # TODO: modify PollingFuture so it can pass a retry argument to done().
--> 528 return super(_AsyncJob, self).result(timeout=timeout)
529
~/anaconda3/envs/env/lib/python3.6/site-packages/google/api_core/future/polling.py in result(self, timeout)
110 # Pylint doesn't recognize that this is valid in this case.
--> 111 raise self._exception
112
BadRequest: 400 Error while reading data, error message: JSON table encountered too many errors, giving up. Rows: 1; errors: 1.
During handling of the above exception, another exception occurred:
GenericGBQException Traceback (most recent call last)
<ipython-input-28-195df93249b6> in <module>()
----> 1 gbq.to_gbq(mini_df,'Name-of-Table','Project-id',chunksize=10000,reauth=False,if_exists='append',private_key=None)
~/anaconda3/envs/env/lib/python3.6/site-packages/pandas/io/gbq.py in to_gbq(dataframe, destination_table, project_id, chunksize, verbose, reauth, if_exists, private_key)
106 chunksize=chunksize,
107 verbose=verbose, reauth=reauth,
--> 108 if_exists=if_exists, private_key=private_key)
~/anaconda3/envs/env/lib/python3.6/site-packages/pandas_gbq/gbq.py in to_gbq(dataframe, destination_table, project_id, chunksize, verbose, reauth, if_exists, private_key, auth_local_webserver)
987 table.create(table_id, table_schema)
988
--> 989 connector.load_data(dataframe, dataset_id, table_id, chunksize)
990
991
~/anaconda3/envs/env/lib/python3.6/site-packages/pandas_gbq/gbq.py in load_data(self, dataframe, dataset_id, table_id, chunksize)
590 job_config=job_config).result()
591 except self.http_error as ex:
--> 592 self.process_http_error(ex)
593
594 rows = []
~/anaconda3/envs/env/lib/python3.6/site-packages/pandas_gbq/gbq.py in process_http_error(ex)
454 # <https://cloud.google.com/bigquery/troubleshooting-errors>`__
455
--> 456 raise GenericGBQException("Reason: {0}".format(ex))
457
458 def run_query(self, query, **kwargs):
GenericGBQException: Reason: 400 Error while reading data, error message: JSON table encountered too many errors, giving up. Rows: 1; errors: 1.

I've had the very same problem.
In my case it depended on the data type object of the Data Frame.
I've had three columns externalId, mappingId, info. For none of those fields I set a data type and let pandas do it's magic.
It decided to set all three column data types to object. Problem is, internally the to_gbq component uses the to_json component. For some reason or another this output omits the quotes around the data field if the type of the field is object but holds only numerical values.
So Google Big Query needed this
{"externalId": "12345", "mappingId":"abc123", "info":"blerb"}
but got this:
{"externalId": 12345, "mappingId":"abc123", "info":"blerb"}
And because the mapping of the field was STRING in Google Big Query, the import process failed.
Two solutions came up.
Solution 1 - Change the data type of the column
A simple type conversion helped with this issue. I also had to change the data type in Big Query to INTEGER.
df['externalId'] = df['externalId'].astype('int')
If this is the case, Big Query can consume fields without quotes as the JSON standard says.
Solution 2 - Make sure the string field is a string
Again, this is setting the data type. But since we set it explicitly to String, the export with to_json prints out a quoted field and everything worked fine.
df['externalId'] = df['externalId'].astype('str')

Disable default behavior - ModelForm Select widget choices filled with referenced objects

I am using a form in Django that is based on a model. So it looks like this:
240 class EventDetailForm(NgFormValidationMixin, NgModelForm):
241 def __init__(self, *args, **kwargs):tha
242 super(EventDetailForm, self).__init__(*args, **kwargs)
243 self.fields['gallery'].queryset = Gallery.objects.none()
244
245 class Meta:
246 model = Event
247 fields = ('title', 'description', 'end_date', 'start_date', 'gallery', 'cover_photo')
248 widgets = {
249 'title': forms.TextInput(attrs={
250 'editable-detail': '',
251 }),
252 'description': forms.TextInput(attrs={
253 'class': 'panel-body',
254 'id': 'event-description-editable',
255 'editable-detail': '',
256 }),
257 'cover_photo': SelectWithDefaultOptions(attrs={
258 'class': 'chosen-select-no-single',
259 'id': 'select-cover-photo',
260 'data-placeholder': 'Select Cover Photo',
261 'style': 'width: 200px;',
262 'tabindex': '-1',
263 }),
264 'start_date': DateTimeWidget(attrs = {
265 'class': 'datetimepicker col-xs-6',
266 'id': 'event-start-date-editable',
267 'editable-detail': '',
268 }),
269 'end_date': DateTimeWidget(attrs = {
270 'class': 'datetimepicker col-xs-6',
271 'id': 'event-end-date-editable',
272 'editable-detail': '',
273 }),
274 'gallery': SelectWithDefaultOptions(attrs={
275 'class': 'chosen-select-no-single',
276 'id': 'select-galley',
277 'data-placeholder': 'Select Gallery',
278 'style': 'width: 200px;',
279 'gallery-select': '',
280 'tabindex': '-1',
281 'organisator-profile-specific': '',
282 }
283 }
So what happens is that my gallery and cover_photo select widgets get filled with all the existing objects of the two types ( because they are actually foreign keys to other models ).
I want to prevent that, and as you see on line 243 I have tried to delete the current queryset ( tried with cleaning choices too resulting in the same ) which works pretty well. The problem is that as you see I use my custom select widget in which I set some default options. It looks like this:
62 class SelectWithDefaultOptions(forms.Select):
63 def __init__(self, attrs=None, choices=()):
64 super(SelectWithDefaultOptions, self).__init__(attrs, choices)
65
66 choices = ('', 'empty') + choices
67 choices = ('None', 'no selection') + choices
The problem is that with the approach I mentioned above I delete those values.
So I said to myself "Well, I will get the needed values, erase all and put the preferred back". Tried it but it turned out that actually the objects that Django puts in are deleting the ones that have been set. ( adds the default ones after the init method has passed)
So I thought "Well, if I set choices=() in the initialisation of the widget (line 274), Django should not set any other values on top of that, because this will violate my choices" so I tried it, but it turned out that Django actually does not care about what choices I would like there to be and act the same.
Also tried to set the field's 'initial' property, still no results.
So, how do I prevent DJango default behviour of putting the referenced objects into the choices list of my select?
Thanks.

This is how I fixed it, with a little help from #AamirAdnan:
65 class SelectWithDefaultOptions(forms.Select):
66 def render(self, name, value, attrs=None, choices=()):
67 choices = ()
68 choices += (('empty', ''),)
69 choices += (('no selection', 'None'),)
70 self.choices = choices
71
72 if value is None:
73 value = ''
74
75 final_attrs = self.build_attrs(attrs, name=name)
76 output = [format_html('<select{0}>', flatatt(final_attrs))]
77 options = self.render_options((), [value])
78
79 if options:
80 output.append(options)
81 output.append('</select>')
82
83 return mark_safe('\n'.join(output))
Just improved the widget render function.
It is not exactly what was asked for, but it works. What happens is:
1. Django sets the choices value to the referenced objects
2. Before the widget is rendered, they are changed to the correct choices

Somehow the widget defined in the Meta is causing this issue. If you just moved it to __init__ it will work. First update your widget, The keyword choices in line def __init__(self, attrs=None, choices=()) means that the choices by default is empty but any instance may override it passing in some values. So you need to set it empty tuple explicitly:
class SelectWithDefaultOptions(forms.Select):
def __init__(self, attrs=None, choices=()):
super(SelectWithDefaultOptions, self).__init__(attrs, choices)
choices = () # explicitly setting choices to empty here
choices += (('', 'empty'),)
choices += (('None', 'no selection'),)
self.choices = choices
Now update your form to assign widget to gallery field in __init__ rather than in Meta class:
class EventDetailForm(NgFormValidationMixin, NgModelForm):
def __init__(self, *args, **kwargs):
super(EventDetailForm, self).__init__(*args, **kwargs)
self.fields['gallery'].widget = SelectWithDefaultOptions(attrs={
'class': 'chosen-select-no-single',
'id': 'select-galley',
'data-placeholder': 'Select Gallery',
'style': 'width: 200px;',
'gallery-select': '',
'tabindex': '-1',
'organisator-profile-specific': '',
}
class Meta:
model = Event
fields = ('title', 'description', 'end_date', 'start_date', 'gallery', 'cover_photo')
OR You don't need any custom widget at all. Just set the choices in forms __init__ method and replace the SelectWithDefaultOptions widget name with forms.Select in Meta (This is cleaner and simpler):
class EventDetailForm(NgFormValidationMixin, NgModelForm):
def __init__(self, *args, **kwargs):
super(EventDetailForm, self).__init__(*args, **kwargs)
self.fields['gallery'].widget.choices = (('', 'Empty',),)
class Meta:
model = Event
fields = ('title', 'description', 'end_date', 'start_date', 'gallery', 'cover_photo')
widgets = {
'gallery': forms.Select(attrs={
'class': 'chosen-select-no-single',
'id': 'select-galley',
'data-placeholder': 'Select Gallery',
'style': 'width: 200px;',
'gallery-select': '',
'tabindex': '-1',
'organisator-profile-specific': '',
}),
}