ssh-keygen -t ed25519 -C "llama-deployment" -f ~/.ssh/llama_do
ssh -i ~/.ssh/llama_do root@YOUR_DROPLET_IP
		

			apt update && apt upgrade -y
curl -fsSL https://get.docker.com -o get-docker.sh
sh get-docker.sh
usermod -aG docker root
docker run hello-world
apt install -y git curl wget htop python3-pip
mkdir -p /opt/llama2-api && cd /opt/llama2-api
		

			FROM python:3.10-slim-bullseye
WORKDIR /app
RUN apt-get update && apt-get install -y \
    build-essential curl git \
    && rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app.py .
RUN mkdir -p /app/models
ARG HF_TOKEN
ENV HF_TOKEN=${HF_TOKEN}
EXPOSE 8000
HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
    CMD curl -f http://localhost:8000/health || exit 1
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]
		

			fastapi==0.104.1
uvicorn==0.24.0
pydantic==2.5.0
torch==2.1.1 --index-url https://download.pytorch.org/whl/cpu
transformers==4.35.2
bitsandbytes==0.41.1
accelerate==0.25.0
peft==0.7.1
		

			from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
import torch, time, logging
from contextlib import asynccontextmanager
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

model = None
tokenizer = None

class GenerationRequest(BaseModel):
    prompt: str
    max_tokens: int = 256
    temperature: float = 0.7
    top_p: float = 0.9
    top_k: int = 50

@asynccontextmanager
async def lifespan(app: FastAPI):
    global model, tokenizer
    logger.info("Loading model...")
    bnb_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_use_double_quant=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=torch.bfloat16
    )
    tokenizer = AutoTokenizer.from_pretrained(
        "meta-llama/Llama-2-7b-hf", cache_dir="/app/models")
    tokenizer.pad_token = tokenizer.eos_token
    model = AutoModelForCausalLM.from_pretrained(
        "meta-llama/Llama-2-7b-hf",
        quantization_config=bnb_config,
        device_map="auto",
        cache_dir="/app/models",
        torch_dtype=torch.bfloat16
    )
    logger.info("Model loaded")
    yield
    del model, tokenizer

app = FastAPI(title="Llama 2 API", version="1.0.0", lifespan=lifespan)

@app.get("/health")
async def health():
    return {"status": "healthy", "model_loaded": model is not None}

@app.post("/generate")
async def generate(request: GenerationRequest):
    if model is None or tokenizer is None:
        raise HTTPException(status_code=503, detail="Model not loaded")
    inputs = tokenizer(request.prompt, return_tensors="pt", truncation=True, max_length=512)
    start = time.time()
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=request.max_tokens,
            temperature=request.temperature,
            top_p=request.top_p,
            top_k=request.top_k,
            do_sample=True,
            pad_token_id=tokenizer.eos_token_id,
            attention_mask=inputs.attention_mask
        )
    inference_time = time.time() - start
    generated_text = tokenizer.decode(
        outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
    return {
        "prompt": request.prompt,
        "generated_text": generated_text,
        "tokens_generated": outputs[0].shape[0] - inputs.input_ids.shape[1],
        "inference_time": inference_time
    }
		

			echo 'HF_TOKEN=hf_ВАШ_ТОКЕН' > /opt/llama2-api/.env
		

			cd /opt/llama2-api
docker build \
  --build-arg HF_TOKEN=$(grep HF_TOKEN .env | cut -d '=' -f2) \
  -t llama2-api:latest .
		

			version: '3.8'
services:
  llama2-api:
    image: llama2-api:latest
    container_name: llama2-api
    ports:
      - "8000:8000"
    volumes:
      - llama-models:/app/models
    environment:
      - HF_TOKEN=${HF_TOKEN}
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 3.5G
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
volumes:
  llama-models:
    driver: local
		

			export HF_TOKEN=$(grep HF_TOKEN .env | cut -d '=' -f2)
docker compose up -d
docker logs -f llama2-api
		

			curl http://localhost:8000/health
curl http://localhost:8000/info
curl -X POST http://localhost:8000/generate \
  -H "Content-Type: application/json" \
  -d '{"prompt": "The future of AI is", "max_tokens": 100}'
		

			apt install -y nginx certbot python3-certbot-nginx
systemctl enable --now nginx
		

			server {
    listen 443 ssl http2;
    server_name llama.example.com;

    ssl_certificate /etc/letsencrypt/live/llama.example.com/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/llama.example.com/privkey.pem;

    location / {
        auth_basic "Restricted";
        auth_basic_user_file /etc/nginx/.htpasswd;
        proxy_pass http://127.0.0.1:8000;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        limit_req zone=api burst=10 nodelay;
    }
}

server {
    listen 80;
    server_name llama.example.com;
    return 301 https://$host$request_uri;
}
		

Часто задаваемые вопросы