+ Email: support@webgenieai.co | sangar@webgenieai.co
+ Discord: https://discord.com/channels/799672011265015819/1316415472563916841 + +--- + +Happy Validating! \ No newline at end of file diff --git a/docs/validator_compute.yml b/docs/validator_compute.yml new file mode 100644 index 00000000..f1fbda6d --- /dev/null +++ b/docs/validator_compute.yml @@ -0,0 +1,38 @@ +# Validator Compute Requirements Specification +# This document outlines the compute requirements for validators. +# It serves as a guide to recommend suitable hardware for your subnet. + +# Note: Specifications for miners may differ from those for validators. + +version: '1.0' # Update this version key as needed to match your release version. + +compute_spec: + + cpu: + min_cores: 32 # Minimum number of CPU cores + min_speed: 2.5 # Minimum speed per core (GHz) + recommended_cores: 32 # Recommended number of CPU cores + recommended_speed: 3.5 # Recommended speed per core (GHz) + architecture: "x86_64" # Architecture type (e.g., x86_64, arm64) + + memory: + min_ram: 120 # Minimum RAM (GB) + min_swap: 4 # Minimum swap space (GB) + recommended_swap: 8 # Recommended swap space (GB) + ram_type: "DDR4" # RAM type (e.g., DDR4, DDR3, etc.) + + storage: + min_space: 500 # Minimum free storage space (GB) + recommended_space: 1000 # Recommended free storage space (GB) + type: "SSD" # Preferred storage type (e.g., SSD, HDD) + min_iops: 1000 # Minimum I/O operations per second (if applicable) + recommended_iops: 5000 # Recommended I/O operations per second + + os: + name: "Ubuntu" # Preferred operating system + version: 20.04 # Preferred operating system version + +network_spec: + bandwidth: + download: 100 # Minimum download bandwidth (Mbps) + upload: 20 # Minimum upload bandwidth (Mbps) \ No newline at end of file diff --git a/docs/webgenie-workflow.png b/docs/webgenie-workflow.png new file mode 100644 index 00000000..d5bf36f3 Binary files /dev/null and b/docs/webgenie-workflow.png differ diff --git a/min_compute.yml b/min_compute.yml deleted file mode 100644 index 1da3bb04..00000000 --- a/min_compute.yml +++ /dev/null @@ -1,87 +0,0 @@ -# Use this document to specify the minimum compute requirements. -# This document will be used to generate a list of recommended hardware for your subnet. - -# This is intended to give a rough estimate of the minimum requirements -# so that the user can make an informed decision about whether or not -# they want to run a miner or validator on their machine. - -# NOTE: Specification for miners may be different from validators - -version: '1.0' # update this version key as needed, ideally should match your release version - -compute_spec: - - miner: - - cpu: - min_cores: 4 # Minimum number of CPU cores - min_speed: 2.5 # Minimum speed per core (GHz) - recommended_cores: 8 # Recommended number of CPU cores - recommended_speed: 3.5 # Recommended speed per core (GHz) - architecture: "x86_64" # Architecture type (e.g., x86_64, arm64) - - gpu: - required: True # Does the application require a GPU? - min_vram: 8 # Minimum GPU VRAM (GB) - recommended_vram: 24 # Recommended GPU VRAM (GB) - cuda_cores: 1024 # Minimum number of CUDA cores (if applicable) - min_compute_capability: 6.0 # Minimum CUDA compute capability - recommended_compute_capability: 7.0 # Recommended CUDA compute capability - recommended_gpu: "NVIDIA A100" # provide a recommended GPU to purchase/rent - - memory: - min_ram: 16 # Minimum RAM (GB) - min_swap: 4 # Minimum swap space (GB) - recommended_swap: 8 # Recommended swap space (GB) - ram_type: "DDR4" # RAM type (e.g., DDR4, DDR3, etc.) - - storage: - min_space: 10 # Minimum free storage space (GB) - recommended_space: 100 # Recommended free storage space (GB) - type: "SSD" # Preferred storage type (e.g., SSD, HDD) - min_iops: 1000 # Minimum I/O operations per second (if applicable) - recommended_iops: 5000 # Recommended I/O operations per second - - os: - name: "Ubuntu" # Name of the preferred operating system(s) - version: 20.04 # Version of the preferred operating system(s) - - validator: - - cpu: - min_cores: 4 # Minimum number of CPU cores - min_speed: 2.5 # Minimum speed per core (GHz) - recommended_cores: 8 # Recommended number of CPU cores - recommended_speed: 3.5 # Recommended speed per core (GHz) - architecture: "x86_64" # Architecture type (e.g., x86_64, arm64) - - gpu: - required: True # Does the application require a GPU? - min_vram: 8 # Minimum GPU VRAM (GB) - recommended_vram: 24 # Recommended GPU VRAM (GB) - cuda_cores: 1024 # Minimum number of CUDA cores (if applicable) - min_compute_capability: 6.0 # Minimum CUDA compute capability - recommended_compute_capability: 7.0 # Recommended CUDA compute capability - recommended_gpu: "NVIDIA A100" # provide a recommended GPU to purchase/rent - - memory: - min_ram: 16 # Minimum RAM (GB) - min_swap: 4 # Minimum swap space (GB) - recommended_swap: 8 # Recommended swap space (GB) - ram_type: "DDR4" # RAM type (e.g., DDR4, DDR3, etc.) - - storage: - min_space: 10 # Minimum free storage space (GB) - recommended_space: 100 # Recommended free storage space (GB) - type: "SSD" # Preferred storage type (e.g., SSD, HDD) - min_iops: 1000 # Minimum I/O operations per second (if applicable) - recommended_iops: 5000 # Recommended I/O operations per second - - os: - name: "Ubuntu" # Name of the preferred operating system(s) - version: 20.04 # Version of the preferred operating system(s) - -network_spec: - bandwidth: - download: 100 # Minimum download bandwidth (Mbps) - upload: 20 # Minimum upload bandwidth (Mbps) diff --git a/neurons/miners/hf_miner.py b/neurons/miners/hf_miner.py new file mode 100644 index 00000000..4ec70a49 --- /dev/null +++ b/neurons/miners/hf_miner.py @@ -0,0 +1,50 @@ +import bittensor as bt + +from webgenie.base.neuron import BaseNeuron +from webgenie.protocol import WebgenieTextSynapse, WebgenieImageSynapse +from webgenie.helpers.images import base64_to_image +from webgenie.utils.gpus import get_gpu_info + +from neurons.miners.hf_models.websight_finetuned import generate_html_from_image + + +def check_requirements(): + total_memory_mb, _, _ = get_gpu_info() + + if total_memory_mb is None: + raise ValueError("No GPU detected. HfMiner requires a GPU.") + + bt.logging.info(f"Total memory: {total_memory_mb}") + + if total_memory_mb < 1024 * 24: + raise ValueError("Insufficient GPU memory. HfMiner requires at least 24GB of GPU memory.") + +check_requirements() + +class HfMiner: + def __init__(self, neuron: BaseNeuron): + self.neuron = neuron + + async def forward_text(self, synapse: WebgenieTextSynapse) -> WebgenieTextSynapse: + raise Exception("Not Supported yet.") + try: + if total_memory_mb > 1024 * 50: + synapse.html = generate_html_from_text(synapse.prompt) + else: + synapse.html = "you don't have enough memory to generate html from text" + return synapse + except Exception as e: + bt.logging.error(f"Error in HfMiner forward_text: {e}") + synapse.html = f"Error in HfMiner forward_text: {e}" + return synapse + + async def forward_image(self, synapse: WebgenieImageSynapse) -> WebgenieImageSynapse: + try: + bt.logging.debug(f"Generating HTML from image") + synapse.html = generate_html_from_image(base64_to_image(synapse.base64_image)) + bt.logging.debug(f"Generated HTML: {synapse.html}") + return synapse + except Exception as e: + bt.logging.error(f"Error in HfMiner forward_image: {e}") + synapse.html = f"Error in HfMiner forward_image: {e}" + return synapse \ No newline at end of file diff --git a/neurons/miners/hf_models/falcon7b.py b/neurons/miners/hf_models/falcon7b.py new file mode 100644 index 00000000..7e47e25d --- /dev/null +++ b/neurons/miners/hf_models/falcon7b.py @@ -0,0 +1,65 @@ +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig + +# Loading original model +model_name = "ybelkada/falcon-7b-sharded-bf16" + +bnb_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4", + bnb_4bit_use_double_quant=True, + bnb_4bit_compute_dtype=torch.float16, +) + +model = AutoModelForCausalLM.from_pretrained( + model_name, + quantization_config=bnb_config, + device_map="auto", + trust_remote_code=True, +) + +tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) +tokenizer.pad_token = tokenizer.eos_token +PEFT_MODEL = "PrincySinghal991/falcon-7b-sharded-bf16-finetuned-html-code-generation" +# PEFT_MODEL = "kasperius/falcon-7b-sharded-bf16-finetuned-html-code-generation-the-css-only" + +peft_model = AutoModelForCausalLM.from_pretrained( + PEFT_MODEL, + quantization_config=bnb_config, + device_map="auto", # Let the transformers library handle device placement + trust_remote_code=True, + torch_dtype=torch.float16, # Use mixed precision to reduce memory usage + low_cpu_mem_usage=True +) + +# Load tokenizer +peft_tokenizer = AutoTokenizer.from_pretrained(PEFT_MODEL, trust_remote_code=True) +peft_tokenizer.pad_token = peft_tokenizer.eos_token + +def generate_html_from_text(prompt): + # Tokenize and generate with the PEFT model + peft_encoding = peft_tokenizer(prompt, return_tensors="pt") + peft_outputs = peft_model.generate( + input_ids=peft_encoding["input_ids"].to(peft_model.device), + attention_mask=peft_encoding["attention_mask"].to(peft_model.device), + max_length=2048, + pad_token_id=peft_tokenizer.eos_token_id, + eos_token_id=peft_tokenizer.eos_token_id + ) + peft_model_html = peft_tokenizer.decode(peft_outputs[0], skip_special_tokens=True) + return peft_model_html[len(prompt):] + +if __name__ == "__main__": + + # Example usage + prompt="create a simple login page with html and css" + print("=================") + import time + start_time = time.time() + print(f"Prompt: {prompt}") + html = generate_html_from_text(prompt) + end_time = time.time() + print(f"Time taken: {end_time - start_time} seconds") + print("=================") + print(html) + diff --git a/neurons/miners/hf_models/websight_finetuned.py b/neurons/miners/hf_models/websight_finetuned.py new file mode 100644 index 00000000..24769dae --- /dev/null +++ b/neurons/miners/hf_models/websight_finetuned.py @@ -0,0 +1,67 @@ +import os +import torch + +from PIL import Image +from transformers import AutoModelForCausalLM, AutoProcessor + +from transformers.image_utils import to_numpy_array, PILImageResampling, ChannelDimension +from transformers.image_transforms import resize, to_channel_dimension_format + +API_TOKEN = os.getenv("HF_TOKEN") +DEVICE = torch.device("cuda") +PROCESSOR = AutoProcessor.from_pretrained( + "HuggingFaceM4/VLM_WebSight_finetuned", + token=API_TOKEN, +) +MODEL = AutoModelForCausalLM.from_pretrained( + "HuggingFaceM4/VLM_WebSight_finetuned", + token=API_TOKEN, + trust_remote_code=True, + torch_dtype=torch.bfloat16, +).to(DEVICE) +image_seq_len = MODEL.config.perceiver_config.resampler_n_latents +BOS_TOKEN = PROCESSOR.tokenizer.bos_token +BAD_WORDS_IDS = PROCESSOR.tokenizer(["
+
+Tech Company
+
+Welcome to Tech Company
+At Tech Company, we are dedicated to providing the best technology solutions for your needs. Our team of experts is always ready to help you with any questions or problems you may have.
+
+
+Tech Company
+
+Welcome to Tech Company
+At Tech Company, we are dedicated to providing the best technology solutions for your needs. Our team of experts is always ready to help you with any questions or problems you may have.
+
+
+Tech Company
+
+Welcome to Tech Company
+At Tech Company, we are dedicated to providing the best technology solutions for your needs. Our team of experts is always ready to help you with any questions or problems you may have.
+Hello, World!
" + html2 = "Hello, World!
tags and trims text inside
tags if the text length exceeds the max limit. + + :param html_content: The HTML content as a string. + :param max_p_count: The maximum number of
tags allowed in any parent tag. + :param max_text_length: The maximum length of text allowed inside
tags. + :return: Modified HTML content with excess
tags removed and text inside
tags shortened. + """ + try: + soup = BeautifulSoup(html_content, 'html.parser') + + # Find all tags that contain
as direct children + for tag in soup.find_all(True): # True will find all tags + # Find only
tags as direct children (not nested
tags) + p_tags = [child for child in tag.find_all('p', recursive=False)] + + if len(p_tags) > max_p_count: + # Remove excess
tags + excess_p_tags = p_tags[max_p_count:] + for p_tag in excess_p_tags: + p_tag.decompose() # Remove the excess
tag + + # Traverse through all
tags and handle text nodes inside them + for p_tag in soup.find_all('p'): # Find all
tags
+ for child in p_tag.contents:
+ if isinstance(child, NavigableString):
+ text_str = str(child)
+ if len(text_str) > max_text_length:
+ shortened = text_str[:max_text_length] + "..." # Shorten the text
+ child.replace_with(shortened) # Replace the original text with the shortened version
+
+ return str(soup)
+ except Exception as e:
+ bt.logging.error(f"Error in shorten_html: {e}")
+ raise e
+
+ async def generate_context(self)->DatasetEntry:
+ try:
+ bt.logging.info("Generating Random Website context")
+ website_url = await self.get_random_website_url()
+ if website_url is None:
+ raise Exception("Failed to get a valid website URL")
+ html = await self.get_rendered_html(website_url)
+ html = await self.shorten_html(html)
+ bt.logging.info(f"Generated website URL: {website_url}")
+ return DatasetEntry(
+ src="random_website",
+ url=f"random_website_{website_url}",
+ ground_truth_html=html,
+ prompt="",
+ base64_image="",
+ )
+ except Exception as e:
+ bt.logging.error(f"Error in generate_context: {e}")
+ raise e
+
diff --git a/webgenie/datasets/synthetic_dataset.py b/webgenie/datasets/synthetic_dataset.py
new file mode 100644
index 00000000..8483d1cc
--- /dev/null
+++ b/webgenie/datasets/synthetic_dataset.py
@@ -0,0 +1,65 @@
+# The paper [Unlocking the conversion of Web Screenshots into HTML Code with the WebSight Dataset]
+# (https://arxiv.org/pdf/2403.09029v1#bib.bib5) is our inspiration.
+# The paper suggests using Mistral-7B-Instruct to generate concepts and use Deepseek-Coder-33b-instruct
+# to generate html, but now we are using openai models here. We are going to use that models on the mainnet
+
+import bittensor as bt
+from typing import List
+from pydantic import BaseModel, Field
+
+from webgenie.datasets.dataset import Dataset, DatasetEntry
+from webgenie.helpers.llms import openai_call
+from webgenie.prompts import PROMPT_GEN_CONCEPT, PROMPT_GEN_HTML
+
+
+class ConceptResponse(BaseModel):
+ concepts: List[str] = Field(description="The concept of the website")
+
+
+class HTMLResponse(BaseModel):
+ html: str = Field(description="The html code of the website")
+
+
+class SyntheticDataset(Dataset):
+ def __init__(self, has_ground_truth_html: bool = True):
+ self.has_ground_truth_html = has_ground_truth_html
+ self.concepts = []
+
+ async def _generate_concepts(self):
+ bt.logging.info("Generating concepts")
+ response = await openai_call(
+ messages = [
+ {"role": "system", "content": PROMPT_GEN_CONCEPT},
+ ],
+ response_format = ConceptResponse,
+ )
+ return response.concepts
+
+ async def _generate_html(self, concept: str):
+ bt.logging.debug(f"Generating HTML from concept: {concept}")
+ response = await openai_call(
+ messages = [
+ {"role": "system", "content": PROMPT_GEN_HTML.format(concept=concept)},
+ ],
+ response_format = HTMLResponse,
+ )
+ return response.html
+
+ async def generate_context(self)->DatasetEntry:
+ bt.logging.info("Generating Synthetic context")
+ if not self.concepts:
+ self.concepts = await self._generate_concepts()
+
+ concept = self.concepts.pop(0)
+
+ if self.has_ground_truth_html == True:
+ ground_truth_html = await self._generate_html(concept)
+ else:
+ ground_truth_html = ""
+
+ return DatasetEntry(
+ src="synthetic",
+ url=f"synthetic_{concept}",
+ prompt=concept,
+ ground_truth_html=ground_truth_html,
+ )
diff --git a/webgenie/helpers/htmls.py b/webgenie/helpers/htmls.py
new file mode 100644
index 00000000..5cae30a5
--- /dev/null
+++ b/webgenie/helpers/htmls.py
@@ -0,0 +1,205 @@
+import bittensor as bt
+import asyncio
+import os
+import re
+import uuid
+
+from bs4 import BeautifulSoup
+from lxml import etree
+from lxml.etree import XMLSyntaxError
+from PIL import Image
+from playwright.async_api import async_playwright
+
+from webgenie.constants import (
+ WORK_DIR,
+ CHROME_HTML_LOAD_TIME,
+ JAVASCRIPT_RUNNING_TIME,
+ PLACE_HOLDER_IMAGE_URL,
+)
+from webgenie.helpers.images import image_to_base64
+
+
+def is_valid_resources(html_content: str) -> bool:
+ """
+ Check if the resources in the HTML content are valid.
+ """
+ # List of allowed patterns for CSS and JavaScript resources
+ allowed_patterns = [
+ r"https?://cdn.jsdelivr.net/npm/tailwindcss@[^/]+/dist/tailwind.min.css",
+ r"https?://stackpath.bootstrapcdn.com/bootstrap/[^/]+/css/bootstrap.min.css",
+ r"https?://code.jquery.com/jquery-[^/]+.min.js",
+ r"https?://stackpath.bootstrapcdn.com/bootstrap/[^/]+/js/bootstrap.bundle.min.js",
+ r"https?://cdnjs.cloudflare.com/ajax/libs/font-awesome/[^/]+/css/font-awesome.min.css",
+ r"https?://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap",
+ ]
+
+ soup = BeautifulSoup(html_content, 'html.parser')
+ resources = soup.find_all(['link', 'script'])
+
+ for resource in resources:
+ if resource.name == 'link' and resource.get('rel') == ['stylesheet']:
+ href = resource.get('href')
+ if href and not any(re.match(pattern, href) for pattern in allowed_patterns):
+ return True
+ elif resource.name == 'script':
+ src = resource.get('src')
+ if src and not any(re.match(pattern, src) for pattern in allowed_patterns):
+ return True
+
+ return True
+
+
+def is_valid_html(html_content: str) -> bool:
+ """
+ Check if the HTML is valid.
+ """
+ try:
+ soup = BeautifulSoup(html_content, 'html.parser')
+ return True
+ except Exception as e:
+ bt.logging.error(f"An error occurred: {e}")
+ return False
+
+
+def seperate_html_css(html_content: str) -> tuple[str, str]:
+ """
+ Seperate the HTML and CSS from the HTML content.
+ """
+ soup = BeautifulSoup(html_content, 'html.parser')
+
+ css = ''
+ for style_tag in soup.find_all('style'):
+ css += style_tag.get_text()
+ for style_tag in soup.find_all('style'):
+ style_tag.decompose()
+
+ head = soup.head
+ if not head:
+ head = soup.new_tag('head')
+ soup.html.insert(0, head)
+
+ link_tag = soup.new_tag('link', rel='stylesheet', href='styles.css')
+ head.append(link_tag)
+ cleaned_html = str(soup)
+ return cleaned_html, css
+
+
+async def html_to_screenshot(html_content: str, page_load_time: int = 1000) -> str:
+ """
+ Take a screenshot of the HTML content.
+ """
+ html_path = f"{WORK_DIR}/screenshot_{uuid.uuid4()}.html"
+ with open(html_path, "w") as f:
+ f.write(html_content)
+ png_path = f"{WORK_DIR}/screenshot_{uuid.uuid4()}.png"
+ url = f"file://{os.path.abspath(html_path)}"
+
+ try:
+ async with async_playwright() as p:
+ # Choose a browser, e.g., Chromium, Firefox, or WebKit
+ browser = await p.chromium.launch()
+ page = await browser.new_page()
+
+ # Navigate to the URL
+ await page.goto(url, timeout=CHROME_HTML_LOAD_TIME)
+
+ await page.wait_for_load_state('networkidle')
+ await page.wait_for_timeout(JAVASCRIPT_RUNNING_TIME)
+
+ # Take the screenshot
+ await page.screenshot(
+ path=png_path,
+ full_page=True,
+ animations="disabled",
+ timeout=CHROME_HTML_LOAD_TIME,
+ )
+ await page.close()
+ await browser.close()
+ except Exception as e:
+ bt.logging.error(f"Failed to take screenshot due to: {e}. Generating a blank image.")
+ # Generate a blank image
+ img = Image.new('RGB', (1280, 960), color = 'white')
+ img.save(png_path)
+
+ await asyncio.sleep(0.1)
+ base64_image = image_to_base64(png_path)
+
+ await asyncio.sleep(0.1)
+ os.remove(html_path)
+ os.remove(png_path)
+ return base64_image
+
+
+def format_html(html_content: str) -> str:
+ """
+ Format the HTML content.
+ """
+ soup = BeautifulSoup(html_content, 'html.parser')
+ return soup.prettify()
+
+
+def replace_image_sources(html_content: str, new_url: str = PLACE_HOLDER_IMAGE_URL) -> str:
+ """
+ Replace the image sources in the HTML content.
+ """
+ soup = BeautifulSoup(html_content, 'html.parser')
+
+ # Replace 'src' attribute in tags
+ for img_tag in soup.find_all('img'):
+ img_tag['src'] = new_url
+
+ # Replace 'srcset' attribute in