diff --git a/docs/source/_static/css/style.css b/docs/source/_static/css/style.css
new file mode 100644
index 0000000..c3e52f4
--- /dev/null
+++ b/docs/source/_static/css/style.css
@@ -0,0 +1,6 @@
+.header-logo {
+    background-image: url("../images/logo.png");
+    background-size: 180px 40px;
+    height: 40px;
+    width: 180px;
+}
diff --git a/docs/source/_static/images/logo.png b/docs/source/_static/images/logo.png
new file mode 100644
index 0000000..0d822aa
Binary files /dev/null and b/docs/source/_static/images/logo.png differ
diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html
new file mode 100644
index 0000000..9ecc159
--- /dev/null
+++ b/docs/source/_templates/layout.html
@@ -0,0 +1,4 @@
+{% extends "!layout.html" %}
+{% block extrahead %}
+    <link href="{{ pathto("_static/css/style.css", True) }}" rel="stylesheet" type="text/css">
+{% endblock %}
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 7d4ef23..4507b70 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -141,17 +141,24 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of plugins themes.
 #
-html_theme = 'sphinx_rtd_theme'
-htmlhelp_basename = 'TreeValue'
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the plugins static files,
-# so a file named "default.css" will overwrite the plugins "default.css".
+import pytorch_sphinx_theme
+html_theme = 'pytorch_sphinx_theme'
+
+html_theme_path = [pytorch_sphinx_theme.get_html_theme_path()]
+html_theme_options = {
+    'logo_url':
+    'https://opendilab.github.io/GenerativeRL/',
+    'menu': [
+        {
+            'name': 'GitHub',
+            'url': 'https://github.com/opendilab/GenerativeRL'
+        },
+    ],
+    # Specify the language of shared menu
+    'menu_lang': "en"
+}
 html_static_path = ['_static']
-
-html_css_files = [
-    'css/custom.css',
-]
+html_css_files = ['css/style.css']
 
 epub_title = project
 epub_exclude_files = ['search.html']
diff --git a/requirements-doc.txt b/requirements-doc.txt
index d839f0d..0f044ec 100644
--- a/requirements-doc.txt
+++ b/requirements-doc.txt
@@ -1,5 +1,5 @@
 Jinja2>=3.0.0
-sphinx>=3.2.0
+sphinx<7
 sphinx_rtd_theme>=0.4.3
 enum_tools
 sphinx-toolbox
@@ -19,3 +19,4 @@ setuptools_scm
 sphinx_autodoc_typehints
 lxml_html_clean
 lxml
+-e git+https://github.com/opendilab/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme