Python Product.scrape方法代码示例

# 需要导入模块: from product import Product [as 别名]
# 或者: from product.Product import scrape [as 别名]

#.........这里部分代码省略.........
						header = row
						# Process column names from header
						index = 0
						name_col = -1
						val_col = -1
						dev_col = -1
						pkg_col = -1
						desc_col = -1
						prod_col = -1
						bom_attribs = {}	# Key = column index, value = name of attribute
						for column in header:
							col = column.lower()
							if 'part' in col or 'name' in col:
								name_col = index
							elif 'value' in col:
								val_col = index
							elif 'device' in col:
								dev_col = index
							elif 'package' in col:
								pkg_col = index
							elif 'description' in col:
								desc_col = index
							elif 'partno' in col or 'partnum' in col or 'part number' in col or 'part#' in col or ('pn' in col and 'vendor' not in col):
								prod_col = index
							else:
								bom_attribs[index] = column
							index += 1
						if name_col == -1:
							raise KeyError("Component name column not found in header!")
						
					else:
						print 'Row: ', row
						#row_attribs = {}
						row_attribs = dict({})
						row_attribs.clear()
						for attrib in bom_attribs.items():
							if len(row[attrib[0]]) > 0:
								row_attribs[attrib[1]] = row[attrib[0]]
						#print 'Row attribs: ', row_attribs
						# No need for a "name column found" check here,
						# as a KeyError would already have been raised above
						new_name = row[name_col]	
						if val_col != -1:
							new_val = row[val_col]
						else:
							new_val = ""
						if dev_col != -1:
							new_dev = row[dev_col]
						else:
							new_dev = ""
						if pkg_col != -1:
							new_pkg = row[pkg_col]
						else:
							new_pkg = ""
						if desc_col != -1:
							new_desc = row[desc_col]
						else:
							new_desc = ""
						if prod_col != -1:
							prod = Product.select_by_pn(row[prod_col], connection)
							if prod is not None and len(prod) > 0:
								new_prod = prod[0]
							else:
								new_prod = None
						else:
							new_prod = None
						
						part = Part(new_name, self, new_val, new_dev, new_pkg, new_desc, new_prod, row_attribs)
						part.product_updater(connection)
						if part.product is None:
							self.parts.append([part.name, part.value, ''])
						else:
							self.parts.append([part.name, part.value, part.product.manufacturer_pn])
					rownum += 1
					
			else:	# No column headers
				for row in reader:
					print row
					# Check for optional product column
					if len(row) == 6:
						if len(row[5]) > 0:
							prod = Product.select_by_pn(row[5], connection)
							if prod is not None and len(prod) > 0:
								part = Part(row[0], self, row[1], row[2], row[3], row[4], prod[0])
							else:
								new_prod = Product('NULL', row[5])
								new_prod.insert(connection)
								new_prod.scrape(connection)
								part = Part(row[0], self, row[1], row[2], row[3], row[4], new_prod)
						else:
							part = Part(row[0], self, row[1], row[2], row[3], row[4], None)
					else:
						part = Part(row[0], self, row[1], row[2], row[3], row[4], None)
					#print 'Got part from CSV: '
					#part.show() 
					part.product_updater(connection)
					if part.product is None:
						self.parts.append([part.name, part.value, ''])
					else:
						self.parts.append([part.name, part.value, part.product.manufacturer_pn])

# 需要导入模块: from product import Product [as 别名]
# 或者: from product.Product import scrape [as 别名]
	def test_csv(self):
		try:
			self.wspace.create_tables()
			other_wspace = Workspace('DB Tests 2', os.path.join(os.getcwd(), 'testfixtures', 'dbtests2.sqlite'))
			other_wspace.create_tables()
			from product import Product, Listing
			from part import Part
			from bom import BOM
			
			other_proj = BOM.new_project('other_proj', 'other', '', other_wspace.memory)
			test_c5_prod = Product('TDK Corporation', 'C1005X5R1V104K', 'general_B11.pdf', 'CAP CER 0.1UF 35V 10% X5R 0402', '0402 (1005 Metric)')
			test_c5_prod.scrape(other_wspace.memory)	# Don't want to add this to the main test DB
			test_c5 = Part('C5', other_proj, '0.1uF', 'C-USC0402', 'C0402', 'CAPACITOR, American symbol', test_c5_prod)
			
			test1_csv = os.path.join(os.getcwd(), 'testfixtures', "test1.csv")
			test2_csv = os.path.join(os.getcwd(), 'testfixtures', "test2.csv")
			test3_csv = os.path.join(os.getcwd(), 'testfixtures', "test3.csv")
			
			self.wspace.projects = self.wspace.list_projects()
			assert len(self.wspace.projects) == 0
			
			test1_bom = BOM.new_project('test1', 'Product column, no header', test1_csv, self.wspace.memory)
			test2_bom = BOM.new_project('test2', 'No product column, no header', test2_csv, self.wspace.memory)
			test3_bom = BOM.new_project('test3', 'Header row, no PN attribute', test3_csv, self.wspace.memory)
			
			self.wspace.projects = self.wspace.list_projects()
			assert len(self.wspace.projects) == 3
			
			
			test1_bom.read_from_file(self.wspace.memory)
			assert len(test1_bom.parts) == 6
			test1_c5_query =  test1_bom.select_parts_by_name('C5', self.wspace.memory)
			assert len(test1_c5_query) == 1
			test1_c5 = test1_c5_query[0]
			assert test1_c5.equals(test_c5, True, True, False, True) == True
			assert test1_c5.equals(test_c5, True, True, True, True) == False
			assert test1_c5.product.equals(test_c5_prod)
			
			test2_bom.read_from_file(self.wspace.memory)
			assert len(test2_bom.parts) == 6
			test2_c5_query =  test2_bom.select_parts_by_name('C5', self.wspace.memory)
			assert len(test2_c5_query) == 1
			test2_c5 = test2_c5_query[0]
			assert test2_c5.project is test2_bom
			#					Check:	Attribs, Name, Proj, Prod
			try:
				assert test2_c5.equals(test_c5, False, True, False, True) == True
			except AssertionError:
				print 'Assertion failed: assert test2_c5.equals(test_c5, False, True, False, True) == True'
				print 'Reference C5: ', test_c5
				print 'Test 1 C5: ', test1_c5
				print 'Test 2 C5: ', test2_c5
				raise AssertionError
			assert test2_c5.equals(test_c5, True, True, False, True) == True
			assert test2_c5.equals(test_c5, True, True, True, True) == False
			assert test2_c5.product.equals(test_c5_prod)
			
			assert test1_bom.parts == test2_bom.parts
			
			test3_bom.read_from_file(self.wspace.memory)
			assert len(test3_bom.parts) == 382
			test3_c5_query =  test3_bom.select_parts_by_name('C5', self.wspace.memory)
			assert len(test3_c5_query) == 1
			test3_c5 = test3_c5_query[0]
			#print 'test3_c5 1st check: \n', test3_c5.show()
			assert test3_c5.equals(test_c5, True, True, False, True) == True
			assert test3_c5.equals(test_c5, True, True, True, True) == False
			assert test3_c5.product.equals(test_c5_prod)
			
			test3_c11_query =  test3_bom.select_parts_by_name('C11', self.wspace.memory)
			assert len(test3_c11_query) == 1
			test3_c11 = test3_c11_query[0]
			assert test3_c5.project is test3_bom
			assert test3_c11.product.equals(test_c5_prod)
			
			c5_prod_query = Product.select_by_pn('C1005X5R1V104K', self.wspace.memory) 
			assert len(c5_prod_query) == 1
			c5_prod = c5_prod_query[0]
			assert c5_prod.equals(test_c5_prod)
			test3_c63_query =  test3_bom.select_parts_by_name('C63', self.wspace.memory)
			# Mystery 'None' printout here... huh?
			try:
				#print 'Trying assertion: assert len(test3_c63_query) == 1'
				assert len(test3_c63_query) == 1
			except AssertionError:
				print 'Assertion failed: assert len(test3_c63_query) == 1'
				print 'len(test3_c63_query): ', len(test3_c63_query)
				for p in test3_c63_query:
					print 'test3_c63_query contains this part: ', p
				raise AssertionError
			test3_c63 = test3_c63_query[0]
			#print '\ntest3_c63: \n', test3_c63.show()	VOLT attribute is being preserved OK
			# C63 has a VOLT = 25V attribute, which C5 does not.
			# Therefore, C63's product should remain unset.
			try:
				#print 'Trying assertion: assert test3_c63.product is None'
				assert test3_c63.product is None
			except AssertionError:
				print 'Assertion failed: assert test3_c63.product is None'
				attribs = []
#.........这里部分代码省略.........

# 需要导入模块: from product import Product [as 别名]
# 或者: from product.Product import scrape [as 别名]
	def product_updater(self, connection, check_wspace=True):
		''' Checks if the Part is already in the DB. 
		Inserts/updates self into DB depending on:
		- The presence of a matching Part in the DB
		- The value of self.product.manufacturer_pn
		- The product of the matching Part in the DB
		Passing an open connection to this method is recommended. '''
		unset_pn = ('', 'NULL', 'none', None, [])
		if(self.is_in_db(connection)):
			#print "Part of same name already in DB"
			old_part = Part.select_by_name(self.name, connection, self.project)[0]
			if self.equals(old_part, True, True, True, False):
				if self.product is not None and self.product.manufacturer_pn not in unset_pn:
					if old_part.product is not None and old_part.product.manufacturer_pn not in unset_pn:
						# TODO: prompt? Defaulting to old_part.product for now (aka do nothing)
						#print 'Matching CSV and DB parts with non-NULL product mismatch, keeping DB version...'
						pass
					elif old_part.product is None or old_part.product.manufacturer_pn in unset_pn:
						self.update(connection)
				elif self.product is None or self.product.manufacturer_pn in unset_pn:
					if old_part.product is not None and old_part.product.manufacturer_pn not in unset_pn:
						pass	# Do nothing in this case
					elif old_part.product is None or old_part.product.manufacturer_pn in unset_pn:
						(candidate_proj_parts, candidate_wspace_parts) = self.find_similar_parts(connection, check_wspace)
						#print 'first find_similar_parts call'
						candidate_products = self.find_matching_products(candidate_proj_parts, candidate_wspace_parts, connection)
						if len(candidate_products) == 0:
							#print 'No matching products found, nothing to do'
							pass
						elif len(candidate_products) == 1:
							self.product = candidate_products[0]
							#print 'Found exactly one matching product, setting product and updating', #self.show()
							self.update(connection)
						else:
							#print 'Found multiple product matches, prompting for selection...'
							# TODO: Currently going with first result, need to prompt for selection
							self.product = candidate_products[0]
							self.update(connection)
						
			else:	# Value/device/package/attribs mismatch
				if self.product is not None and self.product.manufacturer_pn not in unset_pn:
					self.update(connection)
				elif self.product is None or self.product.manufacturer_pn in unset_pn:
					(candidate_proj_parts, candidate_wspace_parts) = self.find_similar_parts(connection, check_wspace)
					#print 'second find_similar_parts call'
					candidate_products = self.find_matching_products(candidate_proj_parts, candidate_wspace_parts, connection)
					if len(candidate_products) == 0:
						#print 'No matching products found, updating as-is'
						pass
					elif len(candidate_products) == 1:
						self.product = candidate_products[0]
						#print 'Found exactly one matching product, setting product and updating'#, self.show()
					else:
						#print 'Found multiple product matches, prompting for selection...'
						# TODO: Currently going with first result, need to prompt for selection
						self.product = candidate_products[0]
					self.update(connection)
		
		else:
			#print 'Part not in DB'
			if self.product is None or self.product.manufacturer_pn in unset_pn:
				(candidate_proj_parts, candidate_wspace_parts) = self.find_similar_parts(connection, check_wspace)
				#print 'third find_similar_parts call'
				candidate_products = self.find_matching_products(candidate_proj_parts, candidate_wspace_parts, connection)
				if len(candidate_products) == 0:
					#print 'No matching products found, inserting as-is'#, self.show()
					pass
				elif len(candidate_products) == 1:
					self.product = candidate_products[0]
					#print 'Found exactly one matching product, setting product and inserting'#, self.show()
				else:
					#print 'Found multiple product matches, prompting for selection...'
					# TODO: Currently going with first result, need to prompt for selection
					self.product = candidate_products[0]
			else:
				if self.product.is_in_db(connection) == False:
					newprod = Product('NULL', self.product.manufacturer_pn)
					newprod.insert(connection)
					newprod.scrape(connection)
			self.insert(connection)