i using this part of code to getting docs from sphinx but sphinx Query always return None. But if i execute code from command line i get correct result, what wrong?
def search_query(query, offset=0):
mode = SPH_MATCH_EXTENDED
host = 'localhost'
port = 9312
index = 'rss_item'
filtercol = 'group_id'
filtervals = []
sortby = '-#weights'
groupby = 'id'
groupsort = '#group desc'
limit = 30
# do query
cl = SphinxClient()
cl.SetServer ( host, port )
cl.SetWeights ( [100, 1] )
cl.SetMatchMode ( mode )
#cl.SetSortMode(SPH_SORT_TIME_SEGMENTS)
if limit:
cl.SetLimits ( offset, limit, max(limit,1000) )
res = cl.Query ( query, index )
docs =[]
for item in res['matches']:
docs.append(item['id'])
return docs
# this is django view
def search(request):
q = request.GET.get('q', '')
offset = int(request.GET.get('older', 0))
docs=search_query(q, offset)
result = Item.objects.filter(id__in=docs).all()
objects = dict([(obj.id, obj) for obj in result])
sorted_objects = [objects[id] for id in docs]
result = sorted_objects
return render_to_response('rss/_items.html',
{'latest_items':result, 'offset':offset+30,'q':q},
context_instance=RequestContext(request))
Server : centos6, sphinx: 2.0.5, django: 1.4.2 , apache/wsgi
resolve issue, by turn Selinux OFF and restart server, i listen to apache process by this command:
ps auxw | grep httpd | awk '{print"-p " $2}' | xargs strace
and see permission denied for access apache to sphinx and memcache
Related
I´m quite new with Airflow and Python. What I´m trying to do is get the result from a Bash command and compose the connection string with that return:
import pyodbc as odbc
import pandas as pd
import datetime as dt
from airflow.operators.bash import BashOperator
from airflow.decorators import dag, task
#dag(schedule_interval='0 15 10 * *', start_date=dt.datetime(2021, 10, 1), catchup=False)
def my_dag2():
wsl_ip = BashOperator(
task_id="wsl_ip",
bash_command="grep -m 1 nameserver /etc/resolv.conf | awk '{print $2}'",
do_xcom_push=True
)
#task()
def run():
def busca_informes_cvm(ano,mes):
url = 'http://dados.cvm.gov.br/dados/FI/DOC/INF_DIARIO/DADOS/inf_diario_fi_{:4d}{:02d}.csv'.format(ano,mes)
return pd.read_csv(url, sep=';')
today = dt.date.today()
ano = (today.year)
mes= (today.month)-1
file_name_Comp = '{:4d}-{:02d}'.format(ano,mes)
file_name = '{:4d}{:02d}.csv'.format(ano,mes)
path_name = r'C:\Airflow\{:4d}{:02d}.csv'.format(ano,mes)
conn = odbc.connect('Driver={ODBC Driver 17 for SQL Server};Server= '+ wsl_ip +';Database=CVM;uid=Airflow;pwd=ubuntu')
df = pd.read_sql_query('select max(DT_COMPTC) from Historico;', conn)
left = df[''].str[:7]
if file_name_Comp <= left[0]:
print('Sair')
else:
informes_diarios = busca_informes_cvm(ano,mes)
informes_diarios.to_csv(file_name, sep=';', index=False)
db_view_nm = '[dbo].[Bulk]'
qry = "BULK INSERT " + db_view_nm + " FROM '" + path_name + "' WITH (FIELDTERMINATOR = ';', ROWTERMINATOR = '0x0a', FIRSTROW = 2,ROWS_PER_BATCH = 100000 )"
cursor = conn.cursor()
success = cursor.execute(qry)
conn.commit()
cursor.close
print('Concluído')
execute = run()
etl_dag = my_dag2()
I need to find a way to convert wsl_ip ia a string. Any help would be appreciated.
To get the output from the "wsl_ip" task, you can use the .output property that's exposed for every operator in Airflow. This property is an abstraction over the classic xcom_pull() method known as an XComArg (see docs here).
You could try something like this:
wsl_ip = BashOperator(
task_id="wsl_ip",
bash_command="grep -m 1 nameserver /etc/resolv.conf | awk '{print $2}'",
do_xcom_push=True
)
#task()
def run(ip):
...
conn = odbc.connect('Driver={ODBC Driver 17 for SQL Server};Server= '+ ip +';Database=CVM;uid=Airflow;pwd=ubuntu')
...
execute = run(ip=wsl_ip.output)
The run TaskFlow function now takes an input as the XCom pushed from your BashOperator task which should be converted to a string. Using the .output in this way also automatically creates a task dependency between the "wsl_ip" and "run" tasks too.
I am working on adding agents to our RSA Authentication Manager (8.X) via PowerShell using the RSA Authentication Manager SDK. I am able to query existing agents without issue using the following code:
$searchAgent = New-Object rsaapi.SearchAgentsCommand
$searchAgent = ($CommandServer.executeCommand($loginCommand.sessionId,$searchAgent)).agents
However, when trying to add a new agent, I get "Cannot convert argument "in1", with value: "rsaapi.AgentDTO", for "executeCommand" to type "rsaapi.TargetableCommand": "Cannot convert the "rsaapi.AgentDTO" value of type "rsaapi.AgentDTO" to type "rsaapi.TargetableCommand"."" returned from the last line. Here is my code snippet:
$hostDTO = New-Object rsaapi.HostDTO
$hostDTO.name = $servername
$hostDTO.primaryIpAddress = $ipAddress
$agentDTO = New-Object rsaapi.AgentDTO
$agentDTO.name = $servername
$agentDTO.primaryAddress = $ipAddress
$agentDTO.agentType = 1
$agentDTO.offlineAuthDataRefreshRequired = $false
$agentDTO.restriction = $false
$agentDTO.securityDomainId = 'ims.securityDomainGUID'
$agentDTO.securityDomainName = 'securityDomainName'
$addAgent = New-Object rsaapi.AddAgentCommand
$addAgent = $CommandServer.executeCommand($loginCommand.sessionId,$agentDTO)
The AdminAPIDemo shows the Python code block below as how to add a new agent:
def createAgent(self, name, addr, alt, sdGuid):
# need a HostDTO to be set
host = HostDTO()
host.setName(name)
host.setPrimaryIpAddress(addr)
host.setSecurityDomainGuid(sdGuid)
host.setNotes("Created by AM Demo code")
# the agent to be created
agent = AgentDTO()
agent.setName(name)
agent.setHost(host)
agent.setPrimaryAddress(addr)
agent.setAlternateAddresses(alt)
agent.setSecurityDomainId(sdGuid)
agent.setAgentType(AgentConstants.STANDARD_AGENT)
agent.setRestriction(1) # only allow activated groups
agent.setEnabled(1)
agent.setOfflineAuthDataRefreshRequired(0)
agent.setNotes("Created by AM Demo code")
cmd = AddAgentCommand(agent)
try:
cmd.execute()
except DuplicateDataException:
print "ERROR: Agent " + name + " already exists."
sys.exit(2)
# return the created agents GUID for further linking
return cmd.getAgentGuid()
Any help would be greatly appreciated!!
Working code to add agents via PowerShell to the RSA Authentication Manager console:
$hostDTO = New-Object rsaapi.HostDTO
$hostDTO.name = $servername
$hostDTO.primaryIpAddress = $ipAddress
$agentDTO = New-Object rsaapi.AgentDTO
$agentDTO.name = $servername
$agentDTO.host = $hostDTO
$agentDTO.primaryAddress = $ipAddress
$agentDTO.agentType = 1
$agentDTO.offlineAuthDataRefreshRequired = $false
$agentDTO.restriction = $true
$agentDTO.securityDomainId = 'ims.securityDomainGUID'
$agentDTO.securityDomainName = 'securityDomainName'
$addAgent = New-Object rsaapi.AddAgentCommand
$addAgent.agentDTO = $agentDTO
$addAgent = $CommandServer.executeCommand($loginCommand.sessionId,$addAgent)
I have a big query table that I need to bring down and populate a MSSQL table with. Since I can't find a BigQuerytoMSSQL operator, I'm doing this by hand.
I've been able to export the table to a series of <>_001.txt, <>_002.txt, etc, and store them into GCS, but now I need to get them down into the Airflow server.
I'm attempting to use the GoogleDownloadOperator, but it seams to have an issue I cannot repair.
Export_to_Local = GoogleCloudStorageDownloadOperator(
task_id='Export_GCS_to_Airflow_Staging',
bucket='offrs',
object='TAX_ASSESSOR_LIVE_*.txt',
filename=Variable.get("temp_directory") + "TAL/*",
google_cloud_storage_conn_id='GCP_Mother_Staging',
dag=dag
)
The above code results in this errror:
google.resumable_media.common.InvalidResponse: ('Request failed with status code', 404, 'Expected one of', <HTTPStatus.OK: 200>, <HTTPStatus.PARTIAL_CONTENT: 206>)
am I missing something? I don't know what the problem is.
THanks
GoogleCloudStorageDownloadOperator does not support wildcards, unfortunately.
The quickest option would be to use gsutil command in BashOperator if your VM is already authorized to that bucket.
The other option is to use the following Custom Operator:
from airflow.contrib.hooks.gcs_hook import GoogleCloudStorageHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from airflow.exceptions import AirflowException
WILDCARD = '*'
class CustomGcsDownloadOperator(BaseOperator):
template_fields = ('source_bucket', 'source_object', 'destination_folder',
'destination_object',)
ui_color = '#f0eee4'
#apply_defaults
def __init__(self,
source_bucket,
source_object,
destination_folder,
destination_object=None,
google_cloud_storage_conn_id='google_cloud_default',
delegate_to=None,
last_modified_time=None,
*args,
**kwargs):
super(CustomGcsDownloadOperator,
self).__init__(*args, **kwargs)
self.source_bucket = source_bucket
self.source_object = source_object
self.destination_folder = destination_folder
self.destination_object = destination_object
self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
self.delegate_to = delegate_to
self.last_modified_time = last_modified_time
def execute(self, context):
hook = GoogleCloudStorageHook(
google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
delegate_to=self.delegate_to
)
if WILDCARD in self.source_object:
total_wildcards = self.source_object.count(WILDCARD)
if total_wildcards > 1:
error_msg = "Only one wildcard '*' is allowed in source_object parameter. " \
"Found {} in {}.".format(total_wildcards, self.source_object)
raise AirflowException(error_msg)
prefix, delimiter = self.source_object.split(WILDCARD, 1)
objects = hook.list(self.source_bucket, prefix=prefix, delimiter=delimiter)
for source_object in objects:
if self.destination_object is None:
destination_object = source_object
else:
destination_object = source_object.replace(prefix,
self.destination_object, 1)
self._download_single_object(hook=hook, source_object=source_object,
destination_object=destination_object)
else:
self._download_single_object(hook=hook, source_object=self.source_object,
destination_object=self.destination_object)
def _download_single_object(self, hook, source_object, destination_object):
if self.last_modified_time is not None:
# Check to see if object was modified after last_modified_time
if hook.is_updated_after(self.source_bucket,
source_object,
self.last_modified_time):
self.log.debug("Object has been modified after %s ", self.last_modified_time)
pass
else:
return
self.log.info('Executing copy of gs://%s/%s to file://%s/%s',
self.source_bucket, source_object,
self.destination_folder, destination_object)
hook.download(self.source_bucket, source_object, destination_object)
This is the sample code I got from the http://snmplabs.com/pysnmp/examples/v3arch/asyncore/agent/cmdrsp/agent-side-mib-implementations.html
here they gave sample program for "Multiple MIB trees under distinct context names".
from pysnmp.entity import engine, config
from pysnmp.entity.rfc3413 import cmdrsp, context
from pysnmp.carrier.asyncore.dgram import udp
from pysnmp.smi import instrum, builder
from pysnmp.proto.api import v2c
# Create SNMP engine
snmpEngine = engine.SnmpEngine()
# Transport setup
# UDP over IPv4
config.addTransport(
snmpEngine,
udp.domainName,
udp.UdpTransport().openServerMode(('127.0.0.1', 161))
)
# SNMPv3/USM setup
# user: usr-md5-none, auth: MD5, priv NONE
config.addV3User(
snmpEngine, 'usr-md5-none',
config.usmHMACMD5AuthProtocol, 'authkey1'
)
# Allow full MIB access for each user at VACM
config.addVacmUser(snmpEngine, 3, 'usr-md5-none', 'authNoPriv', (1, 3, 6, 1, 2, 1), (1, 3, 6, 1, 2, 1))
# Create an SNMP context with default ContextEngineId (same as SNMP engine ID)
snmpContext = context.SnmpContext(snmpEngine)
# Create multiple independent trees of MIB managed objects (empty so far)
mibTreeA = instrum.MibInstrumController(builder.MibBuilder())
mibTreeB = instrum.MibInstrumController(builder.MibBuilder())
# Register MIB trees at distinct SNMP Context names
snmpContext.registerContextName(v2c.OctetString('context-a'), mibTreeA)
snmpContext.registerContextName(v2c.OctetString('context-b'), mibTreeB)
# Register SNMP Applications at the SNMP engine for particular SNMP context
cmdrsp.GetCommandResponder(snmpEngine, snmpContext)
cmdrsp.SetCommandResponder(snmpEngine, snmpContext)
cmdrsp.NextCommandResponder(snmpEngine, snmpContext)
cmdrsp.BulkCommandResponder(snmpEngine, snmpContext)
# Register an imaginary never-ending job to keep I/O dispatcher running forever
snmpEngine.transportDispatcher.jobStarted(1)
# Run I/O dispatcher which would receive queries and send responses
try:
snmpEngine.transportDispatcher.runDispatcher()
except:
snmpEngine.transportDispatcher.closeDispatcher()
raise
I tried it with following snmp walk
snmpwalk -v3 -u usr-md5-none -l authNoPriv -A authkey1 -n context-a 127.0.0.1 .1.3.6
and I am getting
SNMPv2-SMI::dod = No more variables left in this MIB View (It is past the end of the MIB tree)
I understood this is happening because the MIB trees are empty. But how to add my data to it?
To generically add or override a functioning MIB tree's OID, you can add the code below to the code example you provided:
...
mibBuilder = snmpContext.getMibInstrum().getMibBuilder()
MibScalar, MibScalarInstance = mibBuilder.importSymbols(
'SNMPv2-SMI', 'MibScalar', 'MibScalarInstance'
)
class MyStaticMibScalarInstance(MibScalarInstance):
def getValue(self, name, idx):
currentDT = datetime.datetime.now()
return self.getSyntax().clone(
'Hello World! It\'s currently: ' + str(currentDT)
)
mibBuilder.exportSymbols(
'__MY_MIB', MibScalar((1, 3, 6, 1, 2, 1, 1, 1), v2c.OctetString()),
MyStaticMibScalarInstance((1, 3, 6, 1, 2, 1, 1, 1), (0,), v2c.OctetString())
)
# Register SNMP Applications at the SNMP engine for particular SNMP context
cmdrsp.GetCommandResponder(snmpEngine, snmpContext)
...
# snmpwalk -v3 -u usr-md5-none -l authNoPriv -A authkey1 127.0.0.1 .1.3.6
iso.3.6.1.2.1.1.1.0 = STRING: "Hello World! It's currently: 2019-11-18 16:02:43.796005"
iso.3.6.1.2.1.1.2.0 = OID: iso.3.6.1.4.1.20408
iso.3.6.1.2.1.1.3.0 = Timeticks: (494) 0:00:04.94
iso.3.6.1.2.1.1.4.0 = ""
iso.3.6.1.2.1.1.5.0 = ""
iso.3.6.1.2.1.1.6.0 = ""
iso.3.6.1.2.1.1.7.0 = INTEGER: 0
iso.3.6.1.2.1.1.8.0 = Timeticks: (0) 0:00:00.00
iso.3.6.1.2.1.11.1.0 = Counter32: 13
iso.3.6.1.2.1.11.2.0 = Counter32: 0
iso.3.6.1.2.1.11.3.0 = Counter32: 0
iso.3.6.1.2.1.11.4.0 = Counter32: 0
iso.3.6.1.2.1.11.5.0 = Counter32: 0
iso.3.6.1.2.1.11.6.0 = Counter32: 0
iso.3.6.1.2.1.11.8.0 = Counter32: 0
iso.3.6.1.2.1.11.9.0 = Counter32: 0
If you really need that second context and independent tree you can create a generic controller to examine and send back whatever you want.
# Create an SNMP context with default ContextEngineId (same as SNMP engine ID)
snmpContext = context.SnmpContext(snmpEngine)
# Very basic Management Instrumentation Controller without
# any Managed Objects attached. It supports only GET's and
# modded to react to a target OID, otherwise
# always echos request var-binds in response.
class EchoMibInstrumController(instrum.AbstractMibInstrumController):
def readVars(self, varBinds, acInfo=(None, None)):
retItem = []
for ov in varBinds:
if str(ov[0]) == '1.3.6.1.2.1.1.1.0':
currentDT = datetime.datetime.now()
retItem.extend([(ov[0], v2c.OctetString('Hello World! It\'s currently: %s' % str(currentDT)))])
else:
retItem.extend([(ov[0], v2c.OctetString('You queried OID %s' % ov[0]))])
return retItem
# Create multiple independent trees of MIB managed objects (empty so far)
mibTreeA = EchoMibInstrumController()
mibTreeB = instrum.MibInstrumController(builder.MibBuilder())
# Register MIB trees at distinct SNMP Context names
snmpContext.registerContextName(v2c.OctetString('context-a'), mibTreeA)
snmpContext.registerContextName(v2c.OctetString('context-b'), mibTreeB)
snmpget -v3 -u usr-md5-none -l authNoPriv -A authkey1 -n context-a 127.0.0.1 .1.3.6.1.2.1.1.1.0
# snmpget -v3 -u usr-md5-none -l authNoPriv -A authkey1 127.0.0.1 .1.3.6.1.2.1.1.1.0
iso.3.6.1.2.1.1.1.0 = STRING: "PySNMP engine version 4.4.12, Python 3.5.2 (default, Jul 5 2016, 12:43:10) [GCC 5.4.0 20160609]"
# snmpget -v3 -u usr-md5-none -l authNoPriv -A authkey1 -n context-a 127.0.0.1 .1.3.6.1.2.1.1.1.0
iso.3.6.1.2.1.1.1.0 = STRING: "Hello World! It's currently: 2019-11-18 15:56:26.598242"
# snmpget -v3 -u usr-md5-none -l authNoPriv -A authkey1 -n context-a 127.0.0.1 .1.3.6.1.2.1.1.2.0
iso.3.6.1.2.1.1.2.0 = STRING: "You queried OID 1.3.6.1.2.1.1.2.0"
I am very new to python and trying to work on this script which recieves data from multiple ftp sites and download yesterday data according to date directory to my local folder. but if the receives fails on any day it would not update that day records and would go to the next day. I want to sync the files that even if it is missed on particular it should complete sync teh new files to local folder I have tried looking at rsync but need your help to process it on the script.this is my script.
MAX_CHILDREN = 16
ftp_site_prog = "/usr/local/bin/ftp_site.py"
class SpawnQ:
def __init__(self, max_pids):
self.max_pids = max_pids
self.queue = []
tmp = re.split("/", ftp_site_prog)
self.my_name = tmp[-1]
def addQ(self, site_id):
self.queue.append(site_id)
return
def runQ(self):
while (len(self.queue) != 0):
# Check how many sessions are running
cmd = """ps -ef | grep "%s" | grep -v grep""" % self.my_name
num_pids = 0
for line in os.popen(cmd).readlines():
num_pids = num_pids + 1
if (num_pids < self.max_pids):
site_id = self.queue.pop()
# print site_id
# print "Forking........"
fpid = os.fork()
if fpid:
# print "Created child: ", fpid
os.waitpid(fpid, os.WNOHANG)
else:
# print "This is the Child"
# Exec the ftp_site
arg_string = "%s" % site_id
args = [arg_string]
os.execvp(ftp_site_prog, (ftp_site_prog,) + tuple(args))
how to call rsync on my py script.//os.system("rsync -ftp_site_prog, (ftp_site_prog,)+ tuple(args))
sys.exit(0)
else:
# print "Waiting for a spare process...."
time.sleep(10)
return
# Get a list of the sites
db_obj = nprint.celDb()
site_list = db_obj.get_all_site_ids()
myQ = SpawnQ(MAX_CHILDREN)
for site_id in site_list:
myQ.addQ(site_id)
myQ.runQ()
# Wait until we only have the parent left
# Check how many sessions are running
tmp = re.split("/",ftp_site_prog)
ftp_name = tmp[-1]
cmd = """ps -ef | grep "%s" | grep -v grep""" % ftp_name
num_pids = MAX_CHILDREN
while (num_pids > 0):
num_pids = 0
for line in os.popen(cmd).readlines():
num_pids = num_pids + 1
time.sleep(60)
today = datetime.date.today()
daydelta = datetime.timedelta(days=1)
yesterday = today - daydelta
Much of this can be accomplished with the ftplib module for the retrieval of files from standard FTP servers. If you are dealing with SFTP servers you can use the paramiko library.