add wip openwebui local llms guide

This commit is contained in:
oxeo0 2025-04-18 02:21:15 +02:00
parent 4ac4056032
commit 2f9ff33267
37 changed files with 1896 additions and 0 deletions

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 35 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 49 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 54 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 122 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 39 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 21 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 39 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 39 KiB

View file

@ -0,0 +1,53 @@
import matplotlib.pyplot as plt
# Your data: quantization level -> (memory usage, accuracy)
data = {
"Q2_K": (3032, 74.29),
"Q3_K_S": (3495, 82.19),
"Q3_K_M": (3833, 93.29),
"Q4_0": (4460, 96.09),
"Q4_K_S": (4476, 97.38),
"Q4_K_M": (4693, 97.67),
"Q4_1": (4893, 97.18),
"Q5_0": (5354, 98.98),
"Q5_K_S": (5340, 99.08),
"Q5_K_M": (5468, 99.00),
"Q5_1": (5788, 99.16),
"Q6_K": (6291, 99.58),
"Q8_0": (8146, 99.93)
}
# Extract labels, memory usage, and accuracy
labels = list(data.keys())
memory_usage = [value[0] for value in data.values()]
accuracy = [value[1] for value in data.values()]
# Plot setup using a dark theme
plt.style.use('dark_background')
fig, ax1 = plt.subplots()
plt.title('Quantization Levels of llama 3.1 8B')
# Create two y-axes: one for memory usage and the other for accuracy
color_memory = 'tab:cyan'
ax1.set_xlabel('Quantization Level')
ax1.set_ylabel('Memory Usage (MB)', color=color_memory)
ax1.bar(labels, memory_usage, color=color_memory, alpha=0.8, label='Memory Usage')
ax1.tick_params(axis='y', labelcolor=color_memory)
# Second y-axis for accuracy
ax2 = ax1.twinx()
color_accuracy = 'tab:orange'
ax2.set_ylabel('Accuracy (%)', color=color_accuracy)
ax2.plot(labels, accuracy, color=color_accuracy, marker='o', linestyle='-', linewidth=2, markersize=8, label='Accuracy')
ax2.tick_params(axis='y', labelcolor=color_accuracy)
# Adding legends
fig.tight_layout() # To ensure the layout is tight
lines1, labels1 = ax1.get_legend_handles_labels()
lines2, labels2 = ax2.get_legend_handles_labels()
ax1.legend(lines1 + lines2, labels1 + labels2, loc='upper left')
# Show plot
plt.show()

View file

@ -0,0 +1,408 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="">
<meta name="author" content="">
<link rel="shortcut icon" href="../../../../../../assets/img/favicon.png">
<title>Anonymity - Self-Hosted LLM Hidden Service</title>
<!-- Bootstrap core CSS -->
<link href="../../assets/css/bootstrap.css" rel="stylesheet">
<link href="../../assets/css/xt256.css" rel="stylesheet">
<style>
table {
border-collapse: collapse;
max-width: 960px;
}
th, td {
border: 1px solid black;
padding: 10px;
font-family: monospace;
}
tr:nth-child(odd) {
background-color: #0002;
}
th:first-child,
td:first-child,
tr:first-child {
background-color: #0006;
font-weight: bold;
}
</style>
<!-- Custom styles for this template -->
<link href="../../assets/css/main.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
<script src="https://oss.maxcdn.com/libs/respond.js/1.3.0/respond.min.js"></script>
<![endif]-->
</head>
<body>
<!-- Static navbar -->
<div class="navbar navbar-inverse-anon navbar-static-top">
<div class="container">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand-anon" href="\index.html">The Nihilism Opsec Blog</a>
</div>
<div class="navbar-collapse collapse">
<ul class="nav navbar-nav navbar-right">
<li><a href="/about.html">About</a></li>
<li><a href="/blog.html">Categories</a></li>
<li><a href="/contact.html">Contact</a></li>
</ul>
</div><!--/.nav-collapse -->
</div>
</div>
<!-- +++++ First Post +++++ -->
<div id="anon2">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<a href="../index.html">Previous Page</a></br></br><p><img src="../../assets/img/user.png" width="50px" height="50px"> <ba>oxeo0 - 2025 / 04 / 18</ba></p>
<h1>Anonymity - Self-Hosted LLM Hidden Service</h1>
<img src="0.png" style="width:250px">
<img src="1.png" style="width:250px">
<p> </p>
<p><img src="../logos/daturagit.png" style="width:100px"> <u>Sidenote:</u> Help us improve this tutorial by letting us know if there's anything missing or incorrect on this <a href="http://git.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/nihilist/blog-contributions/issues/226">git issue</a> directly!</p>
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /grey -->
<!-- +++++ Second Post +++++ -->
<div id="anon3">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<h2><b>Current state of LLMs</b></h2>
<p>If you've been on the internet recently, there's a high chance you heard about Large Language Models. Most notable companies in this field include OpenAI, Google, Antropic and xAI. To access their models you typically need to communicate with the service via an API. Such use while convenient, means user has little to no knowledge about how the data sent there is stored and used.</p>
<p>Additionally, when users submit data through these services, it might be embedded into future models. Companies often train new models on a variety of user-submitted data, which can include any text inputs you've provided. This raises serious privacy concerns, as personal information could inadvertently become part of the training set for subsequent AI models. AI giants will often say they're trying to respect your privacy with data "anonymization" and other techniques, but we all know how this works in practice. See: <a href="https://privacy.anthropic.com/en/articles/10023555-how-do-you-use-personal-data-in-model-training">Anthropic's Privacy Policy</a> and <a href="https://help.openai.com/en/articles/5722486-how-your-data-is-used-to-improve-model-performance">OpenAI explaining</a> how they "improve model performance" with users data.</p>
<p>The vast amount of sensitive user data stored can have devastating consequences if a leak occurs. In AI space it's not uncommon to leak data either via compromised servers or models themselves. In the past year alone companies suffering such leaks include: <a href="https://www.spiceworks.com/tech/artificial-intelligence/news/chatgpt-leaks-sensitive-user-data-openai-suspects-hack/">OpenAI</a>, <a href="https://venturebeat.com/ai/anthropic-confirms-it-suffered-a-data-leak/">Anthropic</a> and <a href="https://www.wiz.io/blog/wiz-research-uncovers-exposed-deepseek-database-leak">DeepSeek</a>.</p>
<p><b>Assume all conversations with online chatbots can be public at any time.</b></p>
<img src="5.png" style="width:480px">
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /white -->
<div id="anon2">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<h2><b>Privacy LLM frontends</b></h2>
<p>A partial solution to those problems could be a service that aggregates multiple model APIs and anonymizes their users. A bit like <a href="https://github.com/searxng/searxng">searxng</a> does for search engines.<br>
AI companies can't know who exactly uses their models since the amount of metadata is heavily limited.
</p>
<p>
There're several such services including <a href="https://ppq.ai">ppq.ai</a>, <a href="https://nano-gpt.com">NanoGPT</a> or <a href="https://duck.ai">DuckDuckGo chat</a>. This is only a partial solution since your conversation contents can still be saved and used for later training by large AI companies.
</p>
<img src="4.png" style="width:480px">
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /white -->
<div id="anon1">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<h2><b>Open LLMs Primer</b></h2>
<p>
Another option available is to self-host LLM on your own infrastructure. This will effectively prevent sending your data from being sent to third parties. It can work fully offline on device but you'll need to have the required resources. You also have to understand certain more advanced concepts related to LLMs.
</p>
<p>
<b>Parameter Count</b><br>
Each open model has specified number of parameters. This can range from 0.5 billion (qwen 2.5) to even 671 billion (deepseek r1). The more parameters model has, the more knowledge can be packed into it. This comes at a cost of more physical RAM/VRAM memory being used. Newer generation models are fairly capable even at 8 billion parameters but it's not uncommon to use 12, 14 or 32 B ones.
</p>
<p>
<b>Quantization (improving memory usage)</b><br>
Usually the <b>model + context</b> needs to fit into RAM/VRAM memory. Each model parameter can be represented with certain precision. For example, <b>FP16</b> uses 16 bits (2 bytes) of memory to store a single parameter, while <b>Q4_0</b> uses only 4 bits. This means that FP16 model will use ~4x of the memory compared to Q4_0.
Of course using Q4_0 will introduce some rounding error in quantization step, but it's usually not a big deal. Look at the graph below to see how different quantization parameters affect model accuracy and memory usage of llama 3.1 8B.
</p>
<img src="2.png" class="imgRz">
<p>
<br>
I highlighted the <b>Q4_K_S</b> and <b>Q4_K_M</b> quantization methods since they're usually offer the best balance between model size and accuracy.
They usually use a bit more than 4 bits per parameter, but have better precision than plain <b>Q4_0</b>. If you're pulling model from ollama without specifying the precision, there's a high chance that you'll get <b>Q4_K_M</b> variant since it has been the default for some time.
</p>
<p>
The rough formula for calculating memory usage of an Q4_K_M quantized LLM would be: <b>[n billion parameters] * (4.5 / 8) + [context window size]</b>.<br>
For 8B model, we would require around 6 GB VRAM/RAM to comfortably run it as Q4_K_M.
</p>
<p>
<b>Context size</b><br>
Context size is the number of tokens that LLM remembers from previous messages to generate a response. It's usually measured in tokens.<br>
In ollama it's usually set to 2048 tokens, which is around 1200 words or 6 kilobytes of text. With larger sizes, more memory is required to store the context. Also, the models have a context size limit (ex. 16k tokens for Phi-4, 128k for Gemma 3).
If the context size is too small, the LLM may forget what it was doing before. Take a look at this simplified example:<br>
<img src="7.png" class="imgRz"><br><br>
In order to generate a correct response, the entire prompt should fit into the context window:<br>
<img src="8.png" class="imgRz"><br><br>
We'll show how to check prompt length and set appropriate context size in Open WebUI a bit later on.
</p>
<p>
<b>Model recommendations</b><br>
[table that I accidentally deleted...]
</p>
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /white -->
<div id="anon2">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<h2><b>Uses of local AI</b></h2>
<p>Contrary to what companies in the field often say - AI isn't a silver bullet. It won't solve all most problems we face as privacy concious people.<br>However there are some good use-cases even for privacy and anonymity. We already discussed how <a href="../stylometry/index.html">stylometry protection</a> can be achieved with local AI.</p>
<p><b>Translation</b> - LLMs provide high-quality, real-time translations, allowing for communication across languages without external data leaks.<br>
<b>Rewriting</b> - They assist in paraphrasing content to protect against stylometry or improving the flow.<br>
<b>Solving Problems</b> - LLMs can be used as personal assistants to answer every day questions and help with personal issues.<br>
<b>Programming Aid</b> - Developers use them for code suggestions and debugging without exposing their sensitive codebases.</p>
<p>It's crucial to stress that AI can hallucinate (make stuff up). Thus it's never to be fully trusted with anything important. <b>You should always check the information in reputable sources in case of any doubts</b>.</p>
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /white -->
<div id="anon1">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<h2><b>Prerequisites</b></h2>
<p>To follow this tutorial, you'll need a system running Debian 12. Although ollama can work on CPU only, the performance will be much worse than having model that fits in GPU's VRAM.<br>
To comfortably use an 8B model, it's strongly advised to have a dedicated GPU with at least 6GB of VRAM. You can check the supported GPU models <a href="https://github.com/ollama/ollama/blob/main/docs/gpu.md">here</a>.</p>
<p>This tutorial showcases ollama setup with Nvidia drivers, but AMD GPUs are also supported.</p>
<p>If you want to expose Open WebUI via Tor to access it remotely, you should have a <a href="../torwebsite/index.html">hidden service</a> setup.</p>
<p>It's also possible to set this up inside a Proxmox VE or any KVM based VM. You just need to PCI passthrough appropriate GPU inside the <b>Hardware tab</b>:</p>
<img src="6.png" class="imgRz">
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /white -->
<div id="anon2">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<h2><b>Docker Setup</b></h2>
<p>To install Docker, follow the official guide: <a href="https://docs.docker.com/engine/install/debian/">Install Docker Engine on Debian</a>. After installation, add your user to the docker group:</p>
<pre><code>oxeo@andromeda:~$ /sbin/usermod -aG docker oxeo
oxeo@andromeda:~$ sudo systemctl enable docker
</code></pre>
<p>This ensures you can manage Docker without needing sudo privileges. Finally, reboot your system.</p>
<pre><code>oxeo@andromeda:~$ sudo systemctl reboot
</code></pre>
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /white -->
<div id="anon1">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<h2><b>Nvidia Driver and Container Toolkit</b></h2>
<p>Update your package list to include "contrib non-free" at the end of every line in /etc/apt/sources.list:</p>
<pre><code>deb http://deb.debian.org/debian/ bookworm main contrib non-free
deb-src http://deb.debian.org/debian/ bookworm main contrib non-free
deb http://security.debian.org/debian-security bookworm-security main contrib non-free
deb-src http://security.debian.org/debian-security bookworm-security main contrib non-free
</code></pre>
<p>Run:</p>
<pre><code>oxeo@andromeda:~$ sudo apt update
oxeo@andromeda:~$ sudo apt install linux-headers nvidia-driver firmware-misc-nonfree
</code></pre>
<p>To verify installation, execute:</p>
<pre><code>oxeo@andromeda:~$ docker run --rm --runtime=nvidia --gpus all ubuntu nvidia-smi
</code></pre>
<p>This command checks if the NVIDIA driver is accessible within Docker.</p>
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /white -->
<div id="anon2">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<h2><b>Open WebUI Docker Stack</b></h2>
<p>Create a docker-compose.yml file in <b>~/openwebui-stack</b> with the following contents. This setup uses ollama for LLM management and open-webui as the user interface.</p>
<pre><code>services:
ollama:
image: ollama/ollama
container_name: ollama
volumes:
- ollama:/root/.ollama
pull_policy: always
ports:
- 127.0.0.1:11434:11434
tty: true
restart: unless-stopped
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities:
- gpu
open-webui:
image: ghcr.io/open-webui/open-webui:main
container_name: open-webui
volumes:
- open-webui:/app/backend/data
depends_on:
- ollama
ports:
- 127.0.0.1:3000:8080 # Remove "127.0.0.1:" to access from LAN
environment:
- 'OLLAMA_BASE_URL=http://ollama:11434'
- 'WEBUI_SECRET_KEY='
extra_hosts:
- host.docker.internal:host-gateway
restart: unless-stopped
volumes:
ollama: {}
open-webui: {}
</code></pre>
<p>To start the stack:</p>
<pre><code>cd ~/openwebui-stack
docker compose up -d
</code></pre>
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /white -->
<div id="anon1">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<h2><b>Exposing Hidden Service</b></h2>
<p>To expose open-webui via Tor, edit your torrc file:</p>
<pre><code>SocksPort 9050
HiddenServiceDir /var/lib/tor/hidden_service/
HiddenServiceVersion 3
HiddenServicePort 80 127.0.0.1:3000
</code></pre>
<p>Restart Tor and check the generated hostname:</p>
<pre><code>sudo systemctl restart tor
cat /var/lib/tor/hidden_service/hostname
</code></pre>
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /white -->
<div id="anon2">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<h2><b>Troubleshooting</b></h2>
<p>If you encounter issues with hardware acceleration on ollama, check:</p>
<ul>
<li>Ensure the NVIDIA driver is correctly installed and accessible within Docker.</li>
<li>Verify GPU resources are allocated by running `docker run --rm --runtime=nvidia nvidia-smi`.</li>
<li>Check logs with `docker compose logs -f` for any error messages.</li>
</ul>
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /white -->
<div id="anon1">
<div class="container">
<div class="row">
<div class="col-lg-8 col-lg-offset-2">
<h2><b>Closing Remarks</b></h2>
<p>In this tutorial, you've set up a private LLM experience using ollama and open-webui. By exposing it via Tor, your interactions remain anonymous and secure. While consumer-grade hardware may offer less computational power than corporate setups, you retain full control over your data and privacy.</p>
</div>
</div><!-- /row -->
</div> <!-- /container -->
</div><!-- /white -->
<!-- +++++ Footer Section +++++ -->
<div id="anonb">
<div class="container">
<div class="row">
<div class="col-lg-4">
<h4>Nihilism</h4>
<p>
Until there is Nothing left.</p><p><a href="../../../../opsec/legal.html">Legal Disclaimer</a></p><p>Creative Commons Zero: <a href="../../../../opsec/runtheblog/index.html">No Rights Reserved</a></br><img src="\CC0.png">
</p>
</div><!-- /col-lg-4 -->
<div class="col-lg-4">
<h4>My Links</h4>
<p>
<a target="_blank" rel="noopener noreferrer" href="http://blog.nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/rss/feed.xml">RSS Feed</a><br/><a target="_blank" rel="noopener noreferrer" href="http://nowherejezfoltodf4jiyl6r56jnzintap5vyjlia7fkirfsnfizflqd.onion/simplex.html">SimpleX Chatrooms</a><br/>
</p>
</div><!-- /col-lg-4 -->
<div class="col-lg-4">
<h4 class="readable">About nihilist</h4>
<p style="word-wrap: break-word;"><u>Donate XMR:</u>
8AUYjhQeG3D5aodJDtqG499N5jXXM71gYKD8LgSsFB9BUV1o7muLv3DXHoydRTK4SZaaUBq4EAUqpZHLrX2VZLH71Jrd9k8
</p></br>
<p style="word-wrap: break-word;"><u>Donate XMR to the author:</u>
862Sp3N5Y8NByFmPVLTPrJYzwdiiVxkhQgAdt65mpYKJLdVDHyYQ8swLgnVr8D3jKphDUcWUCVK1vZv9u8cvtRJCUBFb8MQ</p>
<p class="readable"><u>Contact:</u> nihilist@contact.nowhere.moe (<a
href="https://nowhere.moe/nihilist.pubkey">PGP</a>)</p>
</div><!-- /col-lg-4 -->
</div>
</div>
</div>
<!-- Bootstrap core JavaScript
================================================== -->
<!-- Placed at the end of the document so the pages load faster -->
</body>
</html>