Python ElasticSearch.index方法代码示例

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
class ESDiffs(object):
    """Implementation of Elastic Search as diff backend"""
    def __init__(self):
        self.es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)

    @staticmethod
    def to_id(label, old, new):
        return "%s/%s/%s" % (label, old, new)

    def put(self, label, old_version, new_version, diff):
        """Store a diff between two versions of a regulation node"""
        struct = {
            'label': label,
            'old_version': old_version,
            'new_version': new_version,
            'diff': diff
        }
        self.es.index(settings.ELASTIC_SEARCH_INDEX, 'diff', struct,
                      id=self.to_id(label, old_version, new_version))

    def get(self, label, old_version, new_version):
        """Find the associated diff"""
        try:
            result = self.es.get(settings.ELASTIC_SEARCH_INDEX, 'diff',
                                 self.to_id(label, old_version, new_version))
            return result['_source']['diff']
        except ElasticHttpNotFoundError:
            return None

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
def update_process_datetime(doc_id, timestamp):
  ''' Updates the last_update_date for the document id passed into function.
    The document id in will be the name of another index in the cluster.
  '''
  connection_string = 'http://localhost:9200'
  process_index = 'openfdametadata'
  _type = 'last_run'
  _map = {}
  _map[_type] = {}
  _map[_type]['properties'] = {}
  _map[_type]['properties']['last_update_date'] = {}
  _map[_type]['properties']['last_update_date']['type'] = 'date'
  _map[_type]['properties']['last_update_date']['format'] = 'dateOptionalTime'

  es = ElasticSearch(connection_string)
  try:
    es.create_index(process_index)
    logging.info('Creating index %s', process_index)
  except exceptions.IndexAlreadyExistsError as e:
    logging.info('%s already exists', process_index)

  try:
    es.put_mapping(process_index, doc_type=_type, mapping=_map)
    logging.info('Successfully created mapping')
  except:
    logging.fatal('Could not create the mapping')

  new_doc = {}
  new_doc['last_update_date'] = timestamp
  es.index(process_index,
           doc_type=_type,
           id=doc_id,
           doc=new_doc,
           overwrite_existing=True)

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
class ESPipeline(object):
    def __init__(self, *args, **kwargs):
        self.client = ElasticSearch('http://localhost:9200/')

    def process_item(self, item, spider):
        self.client.index('wiki', 'page', dict(item))
        return item

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
def main():
    #Train the Naive Bayes Classifier
    f=open('./data_set/naivebayes_trained_model.pickle')
    NBClassifier=pickle.load(f)

    #ElasticSearch- Call the es_indexer file to create 'sentiment_analysis' index and store
    #the contents of the tweet file in that Index
    
    es=ElasticSearch('http://localhost:9200/')
    es_indexer()
    ############Indexing into Elasticsearch############
    i=0
    for each in tweet_data():
        i+=1
        testTweet= each
        processedTestTweet=process_tweet(testTweet)
        sentiment=NBClassifier.classify(extract_features(build_feature_vector(processedTestTweet)))
    
        
        es.index("sentiment_analysis","document",{
                     "text": testTweet,
                     "sentiment": sentiment
                         },id=i)
    print "Indexing completed."

    es.refresh(index="sentiment_analysis")
    print "Index refreshed."

    f.close()

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
def load_pie(
    filename, index_name, type_name, category, name, zone="France", sep=";", display="pie", source="", description=""
):

    f = open(filename, mode="r")
    es = ElasticSearch(CONTEXT["datahub-store"])

    categories = {}
    for line in f:
        key, string_value = line.split(sep, 2)
        value = cjson.decode(string_value)

        categories[key] = value

    serie = {
        "name": name,
        "owner": "public",
        "display": display,
        "zone": zone,
        "category": category,
        "source": source,
        "description": description % (key),
        "data": {"categories": categories.keys(), "series": [{"data": categories.values()}]},
    }
    es.index(index_name, display, serie)

    es.refresh(index_name)
    f.close()

    es.refresh(index_name)
    f.close()

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
class ESNotices(object):
    """Implementation of Elastic Search as notice backend"""
    def __init__(self):
        self.es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)

    def put(self, doc_number, notice):
        """Store a single notice"""
        self.es.index(settings.ELASTIC_SEARCH_INDEX, 'notice', notice,
                      id=doc_number)

    def get(self, doc_number):
        """Find the associated notice"""
        try:
            result = self.es.get(settings.ELASTIC_SEARCH_INDEX, 'notice',
                                 doc_number)

            return result['_source']
        except ElasticHttpNotFoundError:
            return None

    def listing(self, part=None):
        """All notices or filtered by cfr_part"""
        if part:
            query = {'match': {'cfr_part': part}}
        else:
            query = {'match_all': {}}
        query = {'fields': ['effective_on', 'fr_url', 'publication_date'],
                 'query': query}
        notices = []
        results = self.es.search(query, doc_type='notice', size=100,
                                 index=settings.ELASTIC_SEARCH_INDEX)
        for notice in results['hits']['hits']:
            notice['fields']['document_number'] = notice['_id']
            notices.append(notice['fields'])
        return notices

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
class ItvacaturesParseStrategy(ParseStrategy.ParseStrategy):
    def __init__(self):
        self.website = "it-vacatures"
        # elasticsearch binden aan es
        self.es = ElasticSearch("http://localhost:9200/")

    def parseTitel(self, soup):
        titel = soup.head.title.string
        return titel

    def parseWerkgever(self, soup):
        info = soup.find("td")
        infoTwee = info.find_next_sibling()
        p = re.compile(r"<.*?>")
        werkgever = p.sub("", str(infoTwee))
        return werkgever

    def parseLocatie(self, soup):
        info = soup.find("td")
        infoTwee = info.find_next_sibling()
        locatieEen = infoTwee.find_next()
        p = re.compile(r"<.*?>")
        locatieTwee = p.sub("", str(locatieEen))
        p = re.compile(r"Locatie")
        locatie = p.sub("", str(locatieTwee))
        locatie = locatie.strip()
        return locatie

    def parseInhoud(self, soup):
        body = soup.find("div", {"id": "job-description"})
        p = re.compile(r"<.*?>")
        inhoud = p.sub("", str(body))
        return inhoud

    def parse(self, websiteUrl):
        soup = self.getSoup(websiteUrl)

        # parsen
        titel = self.parseTitel(soup)
        try:
            werkgever = self.parseWerkgever(soup)
        except:
            werkgever = "-"
        try:
            locatie = self.parseLocatie(soup)
        except:
            locatie = "-"
        inhoud = self.parseInhoud(soup)
        websiteUrl = re.sub(r"(?s)/\*.*\*/", "", websiteUrl)
        datum = time.strftime("%d-%m-%Y")
        # generate id (string)
        id = self.website + "-" + re.sub(r"\W+", "", titel)

        # make document to be send to elasticsearch database
        document = self.makeDocument(id, titel, websiteUrl, self.website, datum, werkgever, locatie, "-", inhoud)

        # indexeren (stoppen) van vacaturen in esDb
        self.es.index("vacature-index", "vacature", document, id=document["id"])
        print "Es: " + titel

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
class IctergezochtParseStrategy(ParseStrategy.ParseStrategy):
    def __init__(self):
        self.website = "ictergezocht"
        # elasticsearch binden aan es
        self.es = ElasticSearch("http://localhost:9200/")

    def parseWerkgever(self, soup):
        info = soup.find(class_="highlight")
        p = re.compile(r"<.*?>")
        werkgever = p.sub("", str(info))
        return werkgever

    def parseLocatie(self, soup):
        infoTwee = soup.find(class_="bf")
        locatieEen = infoTwee.find_next()
        locatieTwee = locatieEen.find_next()
        locatieDrie = locatieTwee.find_next()
        locatieVier = locatieDrie.find_next()
        p = re.compile(r"<.*?>")
        locatieVijf = p.sub("", str(locatieVier))
        p = re.compile(r"Locatie")
        locatie = p.sub("", str(locatieVijf))
        locatie = locatie.strip()
        return locatie

    def parseInhoud(self, soup):
        body = soup.find(class_="vacancybody")
        p = re.compile(r"<.*?>")
        inhoud = p.sub("", str(body))
        return inhoud

    def parseTitel(self, soup):
        titel = soup.head.title.string
        return titel

    def parse(self, websiteUrl):
        soup = self.getSoup(websiteUrl)

        titel = self.parseTitel(soup)
        if titel.startswith("Vacature"):
            # parsen
            werkgever = self.parseWerkgever(soup)
            locatie = self.parseLocatie(soup)
            inhoud = self.parseInhoud(soup)
            websiteUrl = re.sub(r"(?s)/\*.*\*/", "", websiteUrl)
            datum = time.strftime("%d-%m-%Y")
            # generate id website (string)
            id = self.website + "-" + re.sub(r"\W+", "", titel)

            # make document
            document = self.makeDocument(id, titel, websiteUrl, self.website, datum, werkgever, locatie, "-", inhoud)
            # indexeren (stoppen) van vacaturen in esDb
            self.es.index("vacature-index", "vacature", document, id=document["id"])
            print "Es: " + titel

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
def dump_one_and_one_post_elasticsearch(token):
    es = ElasticSearch('http://localhost:9200/')

    relevant_posts = []

    for element in Newsfeed.newsfeed(token, [], 0, None, 1000):
        if 'from' in element and 'category' in element['from']:
            continue

        post = Post(element, token)

        es.index(token.lower(), "post", post.serialize())

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
def dump_relevant_newsfeed_to_elasticsearch(token):
    es = ElasticSearch('http://localhost:9200/')

    relevant_posts = []

    for element in Newsfeed.newsfeed(token, [], 0, None, 1000):
        if 'from' in element and 'category' in element['from']:
            continue

        post = Post(element, token)
        relevant_posts.append(post.serialize())

    data = {'posts': relevant_posts}
    es.index(token.lower(), "post", data, id=1)

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
class IitjobsParseStrategy(ParseStrategy.ParseStrategy):
    def __init__(self):
        self.website = "iitjobs"
        # elasticsearch binden aan es
        self.es = ElasticSearch('http://localhost:9200/')

    def parseTitel(self, soup):
        titel = soup.head.title.string
        titel = titel.strip()
        return titel

    def parseWerkgever(self, soup):
        body = soup.find("span", {"id": "ctl00_middleContent_idShowJobDetails_lblCompanyName"})
        p = re.compile(r'<.*?>')
        werkgever = p.sub('', str(body))
        werkgever = werkgever.strip()
        return werkgever

    def parseLocatie(self, soup):
        body = soup.find("span", {"id": "ctl00_middleContent_idShowJobDetails_lblCountryID"})
        p = re.compile(r'<.*?>')
        locatie = p.sub('', str(body))
        locatie = locatie.strip()
        return locatie

    def parseInhoud(self, soup):
        body = soup.find("div", {"id": "divJobDescrip"})
        p = re.compile(r'<.*?>')
        inhoud = p.sub('', str(body))
        inhoud = inhoud.strip()
        return inhoud

    def parse(self, websiteUrl):
        soup = self.getSoup(websiteUrl)

        #parsen
        titel = self.parseTitel(soup)
        werkgever = self.parseWerkgever(soup)
        locatie = self.parseLocatie(soup)
        inhoud = self.parseInhoud(soup)
        websiteUrl = re.sub(r'(?s)/\*.*\*/', '', websiteUrl)
        datum = time.strftime("%d-%m-%Y")
        # generate id for website (string)
        id = self.website + "-" + re.sub(r'\W+', '', titel)

        # make document to be send to elasticsearch database
        document = self.makeDocument(id, titel, websiteUrl, self.website, datum, werkgever, locatie, "-", inhoud)
        #indexeren (stoppen) van vacaturen in esDb
        self.es.index('vacature-index', 'vacature', document, id=document['id'])
        print('Es: ' + titel)

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
class ElasticPush(Handler):
    """Posts events to ES."""

    def __init__(self, host='localhost', dest=[]):
        Handler.__init__(self, dest=dest)
        self.es = ElasticSearch('http://%s:9200/' % (host))
        self.source_host = socket.gethostname()

    def push(self, data):
        self.debug("Pushing data %s to elastic search" % (data))
        event = ElasticEvent(data)
        self.es.index("carmanor", "line", event.dict())

        Handler.push(self, data)

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
def resource_create(request, index):
    data = json.loads(request.POST['data'])
    data['new'] = True
    # random identifier
    data['uri'] = ''.join(random.choice('0123456789ABCDEF') for i in range(32))

    # store data in elasticsearch
    es = ElasticSearch(settings.ELASTICSEARCH_URL)
    if index == 'persons':
        es.index(index, "person", data)
    elif index == 'institutes':
        es.index(index, "institute", data)
    es.refresh(index)

    return JsonResponse(data)

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
class SensorDatabase(object):

    def __init__(self, server='http://0.0.0.0:9901'):
        self.server = server
        self.es = ElasticSearch(server)

    def index(self, sensor_id, data):
        return self.es.index('domotic', 'sensor_values', data)

# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import index [as 别名]
class WeatherDatabase(object):

    def __init__(self, server='http://0.0.0.0:9901'):
        self.server = server
        self.es = ElasticSearch(server)

    def index(self, data):
        return self.es.index('weather', 'sensor', data)