debian-reference-2.53/0000755000000000000000000000000012255606611011513 5ustar debian-reference-2.53/datadatepop.ent0000644000000000000000000000005512255345173014514 0ustar debian-reference-2.53/common.ent0000644000000000000000000000056412255345173013523 0ustar debian-reference-2.53/pkgsize.ent0000644000000000000000000005235012255345173013707 0ustar debian-reference-2.53/Makefile0000644000000000000000000004557612255606611013174 0ustar ####################################################################### # Build debian-reference (v2) # vim: set ts=8: ####################################################################### ### key adjustable parameters ####################################################################### # base file name excluding file extension MANUAL := debian-reference # languages translated with PO files LANGPO := ja fr it pt de # languages to skip generation of PDF files (not used now) NOPDF := ja fr it pt de # languages to build document LANGALL = en $(LANGPO) ifndef TMPDIR TMPDIR := $(CURDIR)/tmp endif # Change $(DRAFTMODE) from "yes" to "maybe" when this document # should go into production mode #DRAFTMODE := yes DRAFTMODE := maybe export DRAFTMODE ####################################################################### ### basic constant parameters ####################################################################### # Directories (no trailing slash) DXSL := xslt DBIN := bin DASC := asciidoc DPO := po DIMG := /usr/share/xml/docbook/stylesheet/nwalsh/images # Program name and option XLINT := xmllint --format XPNO := xsltproc --novalid --nonet XPINC := xsltproc --novalid --nonet --xinclude # The threshold should be 80 if translation is completed. THRESHOLD:= 0 TRANSLATE:= po4a-translate -M utf-8 --format docbook --keep $(THRESHOLD) -v GETTEXT := po4a-gettextize -M utf-8 -L utf-8 --format docbook UPDATEPO:= msgmerge --update --previous MSGATTR := msgattrib MSGCAT := msgcat DBLATEX := dblatex # Debian packge archive URL #DEBM := http://ftp.us.debian.org/debian/dists DEBM := http://ftp.jp.debian.org/debian/dists # Debian popcon data source URL UPOPC := http://popcon.debian.org/all-popcon-results.txt.gz # Debian release name and arch used CODE := sid ARCH := amd64 UDEBA := $(DEBM)/$(CODE) UDEBB := $(DEBM)/experimental # cached data removed for package related to remotely fetched data RCACHE := all-popcon-results.txt packages.txt packages.bkup.txt pkg.lst # AsciiDoc source file names in $(DASC) directories for local update ASC_NM := 00_preface.txt 01_tutorial.txt 02_package.txt \ 03_sysinit.txt 04_auth.txt 05_network.txt 06_netapp.txt \ 07_xwindow.txt 08_i18nl10n.txt 09_systips.txt 10_datamngt.txt \ 11_dataconv.txt 12_program.txt 99_appendix.txt \ copyright.txt header1.txt header2.txt $(MANUAL).txt # source asciidoc files (local asciidoc conversion script used for stability) SRC_ASC := $(addprefix $(DASC)/, $(ASC_NM)) # source XML inclusion files (excluding common.ent) ENT_STAT:= datadatepop.ent datadatesize.ent popcon.ent pkgsize.ent ENT_ALL := $(ENT_STAT) common.ent # source PO files for all languages (build prcess requires these) SRC_PO := $(addsuffix .po, $(addprefix $(DPO)/, $(LANGPO))) # source XML files for all languages (build prcess requires these) SRC_XML := $(addsuffix .xml, $(addprefix $(MANUAL)., $(LANGALL))) ####################################################################### # Used as $(call check-command, , ) define check-command set -e; if ! which $(1) >/dev/null; then \ echo "Missing command: $(1), install package: $(2)"; \ false; \ fi endef ####################################################################### # $ make all # build all ####################################################################### .PHONY: all # set LANGPO to limit language to speed up build all: css html txt epub ####################################################################### # $ make test # build html for testing (for Translator) ####################################################################### .PHONY: test test: html css ####################################################################### # $ make publish # build html text from RAWXML/PO for DDP ####################################################################### .PHONY: package # $(PUBLISHDIR) is set to be: /org/www.debian.org/www/doc/manuals for master-www publish: -mkdir -p $(PUBLISHDIR)/$(MANUAL) $(MAKE) css html txt "TMPDIR=$(PUBLISHDIR)/$(MANUAL)" ####################################################################### # $ make clean # clean files ready for tar ####################################################################### .PHONY: clean clean: -rm -f *.swp *~ *.tmp -rm -f $(DPO)/*~ $(DPO)/*.mo $(DPO)/*.po.* -rm -f $(DASC)/*~ -rm -rf $(TMPDIR) $(CURDIR)/tmp -rm -f $(addsuffix .xml, $(addprefix $(MANUAL)., $(LANGALL))) -rm -f $(MANUAL).en.xml $(MANUAL).en.xmlt header.txt -rm -f $(RCACHE) ####################################################################### # $ make distclean # clean files to reset RAWXML/ENT/POT ####################################################################### .PHONY: distclean distclean: clean -rm -f $(MANUAL).raw.xml -rm -f $(ENT_ALL) -rm -f $(DPO)/*.pot -rm -f fuzzy.log ####################################################################### # $ make rawxml # update RAWXML from ASCIIDOC ####################################################################### .PHONY: rawxml rawxml: $(MANUAL).raw.xml $(MANUAL).raw.xml: $(SRC_ASC) @$(call check-command, python, python) @$(call check-command, xmllint, xmllint) $(DBIN)/asciidoc -a 'newline=\n' -o - $(DASC)/$(MANUAL).txt |\ sed -e "/<\/author>/r $(DASC)/copyright.txt" |\ $(XLINT) - |\ sed -f $(DBIN)/replace > $(MANUAL).raw.xml ####################################################################### # $ make entity # update ENT and local cache data from RAWXML/REMOTE ####################################################################### # This ensures not to run remote except by manually $(ENT_STAT): @echo "******** You are missing ENTITY files *******" @echo " $(ENT_STAT)" @echo "*********************************************" @echo " run \"make entity\"" @echo "*********************************************" @false common.ent: $(MANUAL).raw.xml # Release for package match with actual situation echo "" > common.ent echo "" >> common.ent echo "" >> common.ent echo "" >> common.ent echo "" >> common.ent echo "" >> common.ent echo "" >> common.ent echo "" >> common.ent echo "" >> common.ent echo "" >> common.ent .PHONY: entity entity: rawxml @$(call check-command, wget, wget) @$(call check-command, xsltproc, xsltproc) @$(call check-command, grep-dctrl, dctrl-tools) # PACKAGE (sid) wget -O - $(UDEBA)/main/binary-$(ARCH)/Packages.bz2 | bzcat - > packages.main.tmp grep-dctrl -e -sPackage,Installed-Size -P "." packages.main.tmp > packages.main rm packages.main.tmp wget -O - $(UDEBA)/contrib/binary-$(ARCH)/Packages.bz2 | bzcat - > packages.contrib.tmp grep-dctrl -e -sPackage,Installed-Size -P "." packages.contrib.tmp > packages.contrib rm packages.contrib.tmp wget -O - $(UDEBA)/non-free/binary-$(ARCH)/Packages.bz2 | bzcat - > packages.non-free.tmp grep-dctrl -e -sPackage,Installed-Size -P "." packages.non-free.tmp > packages.non-free rm packages.non-free.tmp cat packages.main packages.contrib packages.non-free >packages.txt # echo "" > datadatesize.ent echo "" >> datadatesize.ent echo "" >> datadatesize.ent echo "" >> datadatesize.ent echo "" >> datadatesize.ent rm packages.main packages.contrib packages.non-free # PACKAGE (experimental) wget -O - $(UDEBB)/main/binary-$(ARCH)/Packages.xz | xzcat - > packages.bkup.tmp grep-dctrl -e -sPackage,Installed-Size -P "." packages.bkup.tmp > packages.bkup.txt rm packages.bkup.tmp # PACKAGE LIST of packages mentioned in the source XML $(XPNO) $(DXSL)/pkg.xsl $(MANUAL).raw.xml > pkg.lst # ENT for package size sort pkg.lst | uniq | $(DBIN)/sizeent packages.txt packages.bkup.txt > pkgsize.ent # POPCON wget -O - $(UPOPC) | zcat - > all-popcon-results.txt echo "" > datadatepop.ent echo "" > popcon.ent echo "" >> popcon.ent echo "" >> popcon.ent TOTAL=$$(sed -n -e 's/^Submissions: *\([^ ]*\) *$$/\1/p' < all-popcon-results.txt) ;\ grep -e '^Package:' all-popcon-results.txt | grep -f pkg.lst | $(DBIN)/popconent $$TOTAL >> popcon.ent ####################################################################### # $ make po # update all PO from RAWXML ####################################################################### .PHONY: po pot pot: $(DPO)/templates.pot po: $(SRC_PO) # Do not record line number to avoid useless diff in po/*.po files: --no-location # Do not update templates.pot if contents are the same as before; -I '^"POT-Creation-Date:' $(DPO)/templates.pot: $(MANUAL).en.xmlt FORCE @$(call check-command, po4a-gettextize, po4a) @$(call check-command, msgcat, gettext) $(GETTEXT) -m $(MANUAL).en.xmlt | $(MSGCAT) --no-location -o $(DPO)/templates.pot.new - if diff -I '^"POT-Creation-Date:' -q $(DPO)/templates.pot $(DPO)/templates.pot.new ; then \ echo "Don't update templates.pot" ;\ touch $(DPO)/templates.pot ;\ rm -f $(DPO)/templates.pot.new ;\ else \ echo "Update templates.pot" ;\ mv -f $(DPO)/templates.pot.new $(DPO)/templates.pot ;\ fi : > fuzzy.log # Always update $(DPO)/%.po: $(DPO)/templates.pot FORCE @$(call check-command, msgmerge, gettext) $(UPDATEPO) $(DPO)/$*.po $(DPO)/templates.pot MESS1="no-obsolete $* `$(MSGATTR) --no-obsolete $(DPO)/$*.po |grep ^msgid |sed 1d|wc -l`";\ MESS2="untranslated $* `$(MSGATTR) --untranslated $(DPO)/$*.po |grep ^msgid |sed 1d|wc -l`";\ MESS3="fuzzy $* `$(MSGATTR) --fuzzy $(DPO)/$*.po |grep ^msgid |sed 1d|wc -l`";\ echo "$$MESS1" >>fuzzy.log ; \ echo "$$MESS2" >>fuzzy.log ; \ echo "$$MESS3" >>fuzzy.log ; \ echo "" >>fuzzy.log FORCE: ####################################################################### # $ make wrap # wrap all PO ####################################################################### .PHONY: wrap nowrap wip wrap: @$(call check-command, msgcat, gettext) for XX in $(foreach LX, $(LANGPO), $(DPO)/$(LX).po); do \ $(MSGCAT) -o $$XX $$XX ;\ done nowrap: @$(call check-command, msgcat, gettext) for XX in $(foreach LX, $(LANGPO), $(DPO)/$(LX).po); do \ $(MSGCAT) -o $$XX --no-wrap $$XX ;\ done wip: @$(call check-command, msgattrib, gettext) for XX in $(foreach LX, $(LANGPO), $(DPO)/$(LX).po); do \ $(MSGATTR) -o $$XX.fuzz --fuzzy $$XX ;\ $(MSGATTR) -o $$XX.untr --untranslated $$XX ;\ done ####################################################################### # $ make xml # update all *.XML from RAWXML/ENT/PO/ADD ####################################################################### .PHONY: xml xml: $(SRC_XML) # Update URL list header from $(MANUAL).raw.xml header.txt: $(DASC)/header1.txt $(MANUAL).raw.xml $(DASC)/header2.txt cat $(DASC)/header1.txt > header.txt $(XPNO) $(DXSL)/urls.xsl $(MANUAL).raw.xml | sort | uniq |\ sed -e "s/&/\&/g" >> header.txt cat $(DASC)/header2.txt >> header.txt # Replace table contents with @-@popcon*@-@ and @@@psize*@-@ and # fix URL referencees and table ID. $(MANUAL).en.xml: $(MANUAL).raw.xml header.txt common.ent @$(call check-command, xsltproc, xsltproc) # use asciidoc generated xml file as main contents $(XPNO) $(DXSL)/table.xsl $(MANUAL).raw.xml |\ $(DBIN)/colspec.py |\ sed -e '/ $@ # Replace table contents with dummy text and # fix URL referencees and table ID as the template for translation. # This avoids bloated PO/POT files. (tablet.xsl used insted of table.xsl) $(MANUAL).en.xmlt: $(MANUAL).raw.xml header.txt @$(call check-command, xsltproc, xsltproc) # use asciidoc generated xml file as main contents $(XPNO) $(DXSL)/tablet.xsl $(MANUAL).raw.xml |\ $(DBIN)/colspec.py |\ sed -e '/ $@ $(MANUAL).%.xml: $(DPO)/%.po $(MANUAL).en.xml @$(call check-command, po4a-translate, po4a) @$(call check-command, msgcat, gettext) if [ -f $(DPO)/$*.add ]; then \ $(TRANSLATE) -m $(MANUAL).en.xml -a $(DPO)/$*.add -p $(DPO)/$*.po -l $(MANUAL).$*.xml ;\ else \ $(TRANSLATE) -m $(MANUAL).en.xml -p $(DPO)/$*.po -l $(MANUAL).$*.xml ;\ fi ####################################################################### # $ make css # update CSS and DIMG in $(TMPDIR) ####################################################################### .PHONY: css css: -rm -rf $(TMPDIR)/images mkdir -p $(TMPDIR)/images cp -f $(DXSL)/$(MANUAL).css $(TMPDIR)/$(MANUAL).css echo "AddCharset UTF-8 .txt" > $(TMPDIR)/.htaccess cd $(DIMG) ; cp caution.png home.gif important.png next.gif note.png prev.gif tip.png up.gif warning.png $(TMPDIR)/images ####################################################################### # $ make html # update all HTML in $(TMPDIR) ####################################################################### .PHONY: html html: $(foreach LX, $(LANGALL), $(TMPDIR)/index.$(LX).html) $(TMPDIR)/index.%.html: $(MANUAL).%.xml $(ENT_ALL) @$(call check-command, xsltproc, xsltproc) -mkdir -p $(TMPDIR) $(XPINC) --stringparam base.dir $(TMPDIR)/ \ --stringparam html.ext .$*.html \ $(DXSL)/style-html.xsl $< ####################################################################### # $ make txt # update all Plain TEXT in $(TMPDIR) ####################################################################### .PHONY: txt txt: $(foreach LX, $(LANGALL), $(TMPDIR)/$(MANUAL).$(LX).txt.gz) # style-txt.xsl provides work around for hidden URL links by appending them explicitly. $(TMPDIR)/$(MANUAL).%.txt.gz: $(MANUAL).%.xml $(ENT_ALL) @$(call check-command, w3m, w3m) @$(call check-command, xsltproc, xsltproc) -mkdir -p $(TMPDIR) @test -n "`which w3m`" || { echo "ERROR: w3m not found. Please install the w3m package." ; false ; } $(XPINC) $(DXSL)/style-txt.xsl $< | LC_ALL=en_US.UTF-8 \ w3m -o display_charset=UTF-8 -cols 70 -dump -no-graph -T text/html | \ gzip -9 - > $@ ####################################################################### # $ make pdf # update all PDF in $(TMPDIR) ####################################################################### .PHONY: pdf pdf: $(foreach LX, $(LANGALL), $(TMPDIR)/$(MANUAL).$(LX).pdf) $(foreach LX, $(NOPDF), $(TMPDIR)/$(MANUAL).$(LX).pdf): -mkdir -p $(TMPDIR) echo "PDF generation skipped." >$@ # dblatex.xsl provide work around for hidden URL links by appending them explicitly. $(TMPDIR)/$(MANUAL).%.pdf: $(MANUAL).%.xml $(ENT_ALL) @$(call check-command, dblatex, dblatex) @$(call check-command, xsltproc, xsltproc) -mkdir -p $(CURDIR)/tmp @test -n "`which $(DBLATEX)`" || { echo "ERROR: dblatex not found. Please install the dblatex package." ; false ; } export TEXINPUTS=".:"; \ export TMPDIR="$(CURDIR)/tmp/"; \ $(XPINC) $(DXSL)/dblatex.xsl $< | \ $(DBLATEX) --style=native \ --debug \ --backend=xetex \ --xsl-user=$(DXSL)/user_param.xsl \ --xsl-user=$(DXSL)/xetex_param.xsl \ --param=draft.mode=$(DRAFTMODE) \ --param=lingua=$* \ --output=$@ - || { echo "OMG!!!!!! XXX_CHECK_XXX ... Do not worry ..."; true ; } ####################################################################### # $ make tex # update all TeX source in $(TMPDIR) ####################################################################### .PHONY: tex tex: $(foreach LX, $(LANGALL), $(TMPDIR)/$(MANUAL).$(LX).tex) # dblatex.xsl provide work around for hidden URL links by appending them explicitly. $(TMPDIR)/$(MANUAL).%.tex: $(MANUAL).%.xml $(ENT_ALL) -mkdir -p $(CURDIR)/tmp @test -n "`which $(DBLATEX)`" || { echo "ERROR: dblatex not found. Please install the dblatex package." ; false ; } export TEXINPUTS=".:"; \ export TMPDIR="$(CURDIR)/tmp/"; \ $(XPINC) $(DXSL)/dblatex.xsl $< | \ $(DBLATEX) --style=native \ --debug \ --type=tex \ --backend=xetex \ --xsl-user=$(DXSL)/user_param.xsl \ --xsl-user=$(DXSL)/xetex_param.xsl \ --param=draft.mode=$(DRAFTMODE) \ --param=lingua=$* \ --output=$@ - || { echo "OMG!!!!!! XXX_CHECK_XXX ... Do not worry ..."; true ; } ####################################################################### # $ make epub # update all epub in $(TMPDIR) ####################################################################### .PHONY: epub epub: $(foreach LX, $(LANGALL), $(TMPDIR)/$(MANUAL).$(LX).epub) $(TMPDIR)/$(MANUAL).%.epub: $(MANUAL).%.xml $(ENT_ALL) @$(call check-command, xsltproc, xsltproc) -mkdir -p $(TMPDIR)/$*/ cd $(TMPDIR)/$*/ ; $(XPINC) $(CURDIR)/$(DXSL)/style-epub.xsl $(CURDIR)/$< cp -f $(DXSL)/mimetype $(TMPDIR)/$*/mimetype cp -f $(DXSL)/debian-reference.css $(TMPDIR)/$*/OEBPS/debian-reference.css cp -f $(DXSL)/debian-openlogo.png $(TMPDIR)/$*/OEBPS/debian-openlogo.png cd $(TMPDIR)/$*/ ; zip -r $@ ./ ####################################################################### ### Utility targets ####################################################################### ####################################################################### # $ make rsync # export build result to http://people.debian.org/~osamu/debian-reference/ ####################################################################### .PHONY: rsync rsync: all rsync -avz $(TMPDIR)/ osamu@people.debian.org:public_html/debian-reference/ ####################################################################### # $ make url # check duplicate URL references ####################################################################### .PHONY: url url: $(MANUAL).en.xml @echo "----- Duplicate URL references (start) -----" -sed -ne "/^<\!ENTITY/s/<\!ENTITY \([^ ]*\) .*$$/\" \1 \"/p" < $< | uniq -d | xargs -n 1 grep $< -e | grep -e "^<\!ENTITY" @echo "----- Duplicate URL references (end) -----" ####################################################################### # Translate all ####################################################################### $(DPO)/wikipedia.%.pot: $(DPO)/wikipedia.list $(DBIN)/interwiki $* "PRINT" < $< > $@ ####################################################################### # Translate untranslated ####################################################################### $(DPO)/wikipedia.%.po: $(DPO)/wikipedia.%.list $(DBIN)/interwiki $* "NO" < $< > $@ debian-reference-2.53/fuzzy.log0000644000000000000000000000010012255345173013377 0ustar no-obsolete de 7613 untranslated de 3481 fuzzy de 0 debian-reference-2.53/asciidoc/0000755000000000000000000000000012255345173013274 5ustar debian-reference-2.53/asciidoc/06_netapp.txt0000644000000000000000000017700712255345173015645 0ustar == Network applications // vim: set sts=2 expandtab: // Use ":set nowrap" to edit table After establishing network connectivity (see <<_network_setup>>), you can run various network applications. === Web browsers There are many http://en.wikipedia.org/wiki/Web_Browsers[web browser] packages to access remote contents with http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol[Hypertext Transfer Protocol] (HTTP). .List of web browsers [grid="all"] `------------------`-------------`------------`--------`--------------------------------------------------------------------------------- package popcon size type description of web browser ----------------------------------------------------------------------------------------------------------------------------------------- `chromium` @-@popcon1@-@ @-@psize1@-@ X http://en.wikipedia.org/wiki/Chromium_(web_browser)[Chromium], (open-source browser from Google) `iceweasel` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/Mozilla_Corporation_software_rebranded_by_the_Debian_project[unbranded] http://en.wikipedia.org/wiki/Mozilla_Firefox[Mozilla Firefox] `iceape-browser` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/Mozilla_Corporation_software_rebranded_by_the_Debian_project[unbranded] http://en.wikipedia.org/wiki/Mozilla[Mozilla], removed due to security concerns http://bugs.debian.org/505565[bug#505565] `epiphany-browser` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/GNOME[GNOME], http://en.wikipedia.org/wiki/Human_interface_guidelines[HIG] compliant, http://en.wikipedia.org/wiki/Epiphany_(browser)[Epiphany] `konqueror` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/KDE[KDE], http://en.wikipedia.org/wiki/Konqueror[Konqueror] `dillo` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/Dillo[Dillo], (light weight browser, http://en.wikipedia.org/wiki/FLTK[FLTK] based) `w3m` @-@popcon1@-@ @-@psize1@-@ text http://en.wikipedia.org/wiki/W3m[w3m] `lynx` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/Lynx_(web_browser)[Lynx] `elinks` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/ELinks[ELinks] `links` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/Links_(web_browser)[Links] (text only) `links2` @-@popcon1@-@ @-@psize1@-@ graphics http://en.wikipedia.org/wiki/Links_(web_browser)[Links] (console graphics without X) ----------------------------------------------------------------------------------------------------------------------------------------- // Although I tend to drop reference to removed packages, I keep mozilla as an exception ! ==== Browser configuration You may be able to use following special URL strings for some browsers to confirm their settings. - "`about:`" - "`about:config`" - "`about:plugins`" Debian offers many free browser plugin packages in the main archive area which can handle not only http://en.wikipedia.org/wiki/Java_(software_platform)[Java (software platform)] and http://en.wikipedia.org/wiki/Adobe_Flash[Flash] but also http://en.wikipedia.org/wiki/MPEG-1[MPEG], http://en.wikipedia.org/wiki/MPEG-2[MPEG2], http://en.wikipedia.org/wiki/MPEG-4[MPEG4], http://en.wikipedia.org/wiki/DivX[DivX], http://en.wikipedia.org/wiki/Windows_Media_Video[Windows Media Video (.wmv)], http://en.wikipedia.org/wiki/QuickTime[QuickTime (.mov)], http://en.wikipedia.org/wiki/MP3[MP3 (.mp3)], http://en.wikipedia.org/wiki/Vorbis[Ogg/Vorbis] files, DVDs, VCDs, etc. Debian also offers helper programs to install non-free browser plugin packages as contrib or non-free archive area. .List of browser plugin packages [grid="all"] `------------------------`-------------`------------`---------`--------------------------------------------------------------------------- package popcon size area description ------------------------------------------------------------------------------------------------------------------------------------------ `icedtea-6-plugin` @-@popcon1@-@ @-@psize1@-@ main Java plugin based on OpenJDK and IcedTea `mozilla-plugin-gnash` @-@popcon1@-@ @-@psize1@-@ main Flash plugin based on Gnash `flashplugin-nonfree` @-@popcon1@-@ @-@psize1@-@ contrib Flash plugin helper to install Adobe Flash Player (i386, amd64 only) `browser-plugin-vlc` @-@popcon1@-@ @-@psize1@-@ main Multimedia plugin based on http://en.wikipedia.org/wiki/VLC_media_player[VLC media player] `totem-mozilla` @-@popcon1@-@ @-@psize1@-@ main Multimedia plugin based on http://en.wikipedia.org/wiki/Totem_(media_player)[GNOME@@@sq@@@s Totem media player] `gecko-mediaplayer` @-@popcon1@-@ @-@psize1@-@ main Multimedia plugin based on (GNOME) http://en.wikipedia.org/wiki/MPlayer[MPlayer] ------------------------------------------------------------------------------------------------------------------------------------------ TIP: Although use of above Debian packages are much easier, browser plugins can be still manually enabled by installing "\*.so" into plugin directories (e.g., "`/usr/lib/iceweasel/plugins/`") and restarting browsers. Some web sites refuse to be connected based on the user-agent string of your browser. You can work around this situation by http://www.mozilla.org/unix/customizing.html#prefs[spoofing the user-agent string]. For example, you can do this by adding following line into user configuration files such as "`\~/.gnome2/epiphany/mozilla/epiphany/user.js`" or "`\~/.mozilla/firefox/\*.default/user.js`". -------------------- user_pref{"general.useragent.override","Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)"}; -------------------- Alternatively, you can add and reset this variable by typing "`about:config`" into URL and right clicking its display contents. CAUTION: Spoofed user-agent string may cause https://bugzilla.mozilla.org/show_bug.cgi?id=83376[bad side effects with Java]. === The mail system CAUTION: If you are to set up the mail server to exchange mail directly with the Internet, you should be better than reading this elementary document. NOTE: The following configuration examples are only valid for the typical mobile workstation on consumer grade Internet connections. ==== Email basics An http://en.wikipedia.org/wiki/Email[email] message consists of three components, the message envelope, the message header, and the message body. The "To" and "From" information in the message envelope is used by the http://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol[SMTP] to deliver the email. (The "From" information in the message envelope is also called http://en.wikipedia.org/wiki/Bounce_address[bounce address], From_, etc.) The "To" and "From" information in the message header is displayed by the http://en.wikipedia.org/wiki/Email_client[email client]. (While it is most common for these to be the same as ones in the message envelope, such is not always the case.) The http://en.wikipedia.org/wiki/Email_client[email client] needs to interpret the message header and body data using http://en.wikipedia.org/wiki/MIME[Multipurpose Internet Mail Extensions (MIME)] to deal the content data type and encoding. ==== Modern mail service basics In order to contain spam (unwanted and unsolicited email) problems, many ISPs which provide consumer grade Internet connections are implementing counter measures. - The smarthost service for their customers to send message uses the message submission port (587) specified in http://tools.ietf.org/html/rfc4409[rfc4409] with the password (http://en.wikipedia.org/wiki/SMTP-AUTH[SMTP AUTH] service) specified in http://tools.ietf.org/html/rfc4954[rfc4954]. - The http://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol[SMTP] port (25) connection from their internal network hosts (except ISP@@@sq@@@s own outgoing mail server) to the Internet are blocked. - The http://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol[SMTP] port (25) connection to the ISP@@@sq@@@s incoming mail server from some suspicious external network hosts are blocked. (The connection from hosts on the dynamic IP address range used by the dial-up and other consumer grade Internet connections are the first ones to be blocked.) - http://en.wikipedia.org/wiki/Anti-spam_techniques[Anti-spam techniques] such as http://en.wikipedia.org/wiki/DomainKeys_Identified_Mail[DomainKeys Identified Mail (DKIM)] and http://en.wikipedia.org/wiki/Sender_Policy_Framework[Sender_Policy_Framework (SPF)] are widely used for the http://en.wikipedia.org/wiki/Email_filtering[email filtering]. - The http://en.wikipedia.org/wiki/DomainKeys_Identified_Mail[DomainKeys Identified Mail] service may be provided for your mail sent through the smarthost. When configuring your mail system or resolving mail delivery problems, you must consider these new limitations. In light of these hostile Internet situation and limitations, some independent Internet mail ISPs such as Yahoo.com and Gmail.com offer the secure mail service which can be connected from anywhere on the Internet using http://en.wikipedia.org/wiki/Transport_Layer_Security[Transport Layer Security (TLS) and its predecessor, Secure Sockets Layer (SSL)]. - The smarthost service on port 465 with the deprecated SMTP over SSL (http://en.wikipedia.org/wiki/SMTPS[SMTPS] protocol). - The smarthost service on port 587 with the http://en.wikipedia.org/wiki/STARTTLS[STARTTLS]. - The incoming mail is accessible at the TLS/POP3 port (995) with http://en.wikipedia.org/wiki/Post_Office_Protocol[POP3]. CAUTION: It is not realistic to run SMTP server on consumer grade network to send mail directly to the remote host reliably. They are very likely to be rejected. You must use some smarthost services offered by your connection ISP or independent mail ISPs. For the simplicity, I assume that the smarthost is located at "`smtp.hostname.dom`", requires http://en.wikipedia.org/wiki/SMTP_Authentication[SMTP Authentication], and uses the message submission port (587) with the http://en.wikipedia.org/wiki/STARTTLS[STARTTLS] in the following text. ==== The mail configuration strategy for workstation The most simple mail configuration is that the mail is sent to the ISP@@@sq@@@s smarthost and received from ISP@@@sq@@@s POP3 server by the MUA (see <<_mail_user_agent_mua>>) itself. This type of configuration is popular with full featured GUI based MUA such as `icedove`(1), `evolution`(1), etc. If you need to filter mail by their types, you use MUA@@@sq@@@s filtering function. For this case, the local MTA (see <<_mail_transport_agent_mta>>) need to do local delivery only. The alternative mail configuration is that the mail is sent via local MTA to the ISP@@@sq@@@s smarthost and received from ISP@@@sq@@@s POP3 by the mail retriever (see <<_the_remote_mail_retrieval_and_forward_utility>>) to the local mailbox. If you need to filter mail by their types, you use MDA with filter (see <<_mail_delivery_agent_mda_with_filter>>) to filter mail into separate mailboxes. This type of configuration is popular with simple console based MUA such as `mutt`(1), `mew`(1), etc., although this is possible with any MUAs (see <<_mail_user_agent_mua>>). For this case, the local MTA (see <<_mail_transport_agent_mta>>) need to do both smarthost delivery and local delivery. Since mobile workstation does not have valid FQDN, you must configure the local MTA to hide and spoof the real local mail name in outgoing mail to avoid mail delivery errors (see <<_the_mail_address_configuration>>). TIP: You may wish to configure MUA/MDA to use http://en.wikipedia.org/wiki/Maildir[Maildir] for storing email messages somewhere under your home directory. === Mail transport agent (MTA) For normal workstation, the popular choice for Mail transport agent (MTA) is either `exim4-\*` or `postfix` packages. It is really up to you. .List of basic mail transport agent related packages for workstation [grid="all"] `--------------------`-------------`------------`-------------------------------------------------------------------- package popcon size description --------------------------------------------------------------------------------------------------------------------- `exim4-daemon-light` @-@popcon1@-@ @-@psize1@-@ Exim4 mail transport agent (MTA: Debian default) `exim4-base` @-@popcon1@-@ @-@psize1@-@ Exim4 documentation (text) and common files `exim4-doc-html` @-@popcon1@-@ @-@psize1@-@ Exim4 documentation (html) `exim4-doc-info` @-@popcon1@-@ @-@psize1@-@ Exim4 documentation (info) `postfix` @-@popcon1@-@ @-@psize1@-@ Postfix mail transport agent (MTA: alternative) `postfix-doc` @-@popcon1@-@ @-@psize1@-@ Postfix documentation (html+text) `sasl2-bin` @-@popcon1@-@ @-@psize1@-@ Cyrus SASL API implementation (supplement postfix for SMTP AUTH) `cyrus-sasl2-doc` @-@popcon1@-@ @-@psize1@-@ Cyrus SASL - documentation --------------------------------------------------------------------------------------------------------------------- Although the popcon vote count of `exim4-\*` looks several times popular than that of `postfix`, this does not mean `postfix` is not popular with Debian developers. The Debian server system uses both `exim4` and `postfix`. The http://wiki.debian.org/DefaultMTA[mail header analysis] of mailing list postings from prominent Debian developers also indicate both of these MTAs are as popular. The `exim4-\*` packages are known to have very small memory consumption and very flexible for its configuration. The `postfix` package is known to be compact, fast, simple, and secure. Both come with ample documentation and are as good in quality and license. There are many choices for mail transport agent (MTA) packages with different capability and focus in Debian archive. .List of choices for mail transport agent (MTA) packages in Debian archive [grid="all"] `--------------------`-------------`------------`--------------------------------------------------- package popcon size capability and focus ---------------------------------------------------------------------------------------------------- `exim4-daemon-light` @-@popcon1@-@ @-@psize1@-@ full `postfix` @-@popcon1@-@ @-@psize1@-@ full (security) `exim4-daemon-heavy` @-@popcon1@-@ @-@psize1@-@ full (flexible) `sendmail-bin` @-@popcon1@-@ @-@psize1@-@ full (only if you are already familiar) `nullmailer` @-@popcon1@-@ @-@psize1@-@ strip down, no local mail `ssmtp` @-@popcon1@-@ @-@psize1@-@ strip down, no local mail `courier-mta` @-@popcon1@-@ @-@psize1@-@ very full (web interface etc.) `xmail` @-@popcon1@-@ @-@psize1@-@ light `masqmail` @-@popcon1@-@ @-@psize1@-@ light `esmtp` @-@popcon1@-@ @-@psize1@-@ light `esmtp-run` @-@popcon1@-@ @-@psize1@-@ light (sendmail compatibility extension to `esmtp`) `msmtp` @-@popcon1@-@ @-@psize1@-@ light `msmtp-mta` @-@popcon1@-@ @-@psize1@-@ light (sendmail compatibility extension to `msmtp`) ---------------------------------------------------------------------------------------------------- ==== The configuration of exim4 For the Internet mail via smarthost, you (re)configure `exim4-\*` packages as the following. -------------------- $ sudo /etc/init.d/exim4 stop $ sudo dpkg-reconfigure exim4-config -------------------- Select "mail sent by smarthost; received via SMTP or fetchmail" for "General type of mail configuration". Set "System mail name:" to its default as the FQDN (see <<_the_hostname_resolution>>). Set "IP-addresses to listen on for incoming SMTP connections:" to its default as "127.0.0.1 ; ::1". Unset contents of "Other destinations for which mail is accepted:". Unset contents of "Machines to relay mail for:". Set "IP address or host name of the outgoing smarthost:" to "smtp.hostname.dom:587". Select "" for "Hide local mail name in outgoing mail?". (Use "`/etc/email-addresses`" as in <<_the_mail_address_configuration>>, instead.) Reply to "Keep number of DNS-queries minimal (Dial-on-Demand)?" as one of the following. - "No" if the system is connected to the Internet while booting. - "Yes" if the system is **not** connected to the Internet while booting. Set "Delivery method for local mail:" to "mbox format in /var/mail/". Select "" for "Split configuration into small files?:". Create password entries for the smarthost by editing "`/etc/exim4/passwd.client`". -------------------- $ sudo vim /etc/exim4/passwd.client ... $ cat /etc/exim4/passwd.client ^smtp.*\.hostname\.dom:username@hostname.dom:password -------------------- Start `exim4` by the following. -------------------- $ sudo /etc/init.d/exim4 start -------------------- The host name in "`/etc/exim4/passwd.client`" should not be the alias. You check the real host name with the following. -------------------- $ host smtp.hostname.dom smtp.hostname.dom is an alias for smtp99.hostname.dom. smtp99.hostname.dom has address 123.234.123.89 -------------------- I use regex in "`/etc/exim4/passwd.client`" to work around the alias issue. SMTP AUTH probably works even if the ISP moves host pointed by the alias. You can manually updating `exim4` configuration by the following: - Update `exim4` configuration files in "`/etc/exim4/`". * creating "`/etc/exim4/exim4.conf.localmacros`" to set MACROs and editing "`/etc/exim4/exim4.conf.template`". (non-split configuration) * creating new files or editing existing files in the "`/etc/exim4/exim4.conf.d`" subdirectories. (split configuration) - Run "`invoke-rc.d exim4 reload`". Please read the official guide at: "`/usr/share/doc/exim4-base/README.Debian.gz`" and `update-exim4.conf`(8). CAUTION: Starting `exim4` takes long time if "No" (default value) was chosen for the debconf query of "Keep number of DNS-queries minimal (Dial-on-Demand)?" and the system is **not** connected to the Internet while booting. WARNING: It is insecure to use plain text password without encryption even if your ISP allows it. TIP: Although use of http://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol[SMTP] with http://en.wikipedia.org/wiki/STARTTLS[STARTTLS] on port 587 is recommended, some ISPs still use deprecated http://en.wikipedia.org/wiki/SMTPS[SMTPS] (SSL on port 465). Exim4 after 4.77 supports this deprecated SMTPS protocol for both as client and as server. TIP: If you are looking for a light weight MTA that respects "`/etc/aliases`" for your laptop PC, you should consider to configure `exim4`(8) with "`QUEUERUNNER=@@@sq@@@queueonly@@@sq@@@`", "`QUEUERUNNER=@@@sq@@@nodaemon@@@sq@@@`", etc. in "`/etc/default/exim4`". ==== The configuration of postfix with SASL For the Internet mail via smarthost, you should first read http://www.postfix.org/documentation.html[postfix documentation] and key manual pages. .List of important postfix manual pages [grid="all"] `--------------`---------------------------------- command function -------------------------------------------------- `postfix`(1) Postfix control program `postconf`(1) Postfix configuration utility `postconf`(5) Postfix configuration parameters `postmap`(1) Postfix lookup table maintenance `postalias`(1) Postfix alias database maintenance -------------------------------------------------- You (re)configure `postfix` and `sasl2-bin` packages as follows. -------------------- $ sudo /etc/init.d/postfix stop $ sudo dpkg-reconfigure postfix -------------------- Chose "Internet with smarthost". Set "SMTP relay host (blank for none):" to "`[smtp.hostname.dom]:587`" and configure it by the following. -------------------- $ sudo postconf -e 'smtp_sender_dependent_authentication = yes' $ sudo postconf -e 'smtp_sasl_auth_enable = yes' $ sudo postconf -e 'smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd' $ sudo postconf -e 'smtp_sasl_type = cyrus' $ sudo vim /etc/postfix/sasl_passwd -------------------- Create password entries for the smarthost. -------------------- $ cat /etc/postfix/sasl_passwd [smtp.hostname.dom]:587 username:password $ sudo postmap hush:/etc/postfix/sasl_passwd -------------------- Start the `postfix` by the following. -------------------- $ sudo /etc/init.d/postfix start -------------------- Here the use of "`[`" and "`]`" in the `dpkg-reconfigure` dialog and "`/etc/postfix/sasl_passwd`" ensures not to check MX record but directly use exact hostname specified. See "Enabling SASL authentication in the Postfix SMTP client" in "`/usr/share/doc/postfix/html/SASL_README.html`". ==== The mail address configuration There are a few http://www.debian.org/doc/debian-policy/ch-customized-programs#s-mail-transport-agents[mail address configuration files for mail transport, delivery and user agents]. .List of mail address related configuration files [grid="all"] `----------------------`-------------------------------------`---------------------------------------------------------------------- file function application ------------------------------------------------------------------------------------------------------------------------------------ `/etc/mailname` default host name for (outgoing) mail Debian specific, `mailname`(5) `/etc/email-addresses` host name spoofing for outgoing mail `exim`(8) specific, `exim4-config_files`(5) `/etc/postfix/generic` host name spoofing for outgoing mail `postfix`(1) specific, activated after `postmap`(1) command execution. `/etc/aliases` account name alias for incoming mail general, activated after `newaliases`(1) command execution. ------------------------------------------------------------------------------------------------------------------------------------ The **mailname** in the "`/etc/mailname`" file is usually a fully qualified domain name (FQDN) that resolves to one of the host@@@sq@@@s IP addresses. For the mobile workstation which does not have a hostname with resolvable IP address, set this **mailname** to the value of "`hostname -f`". (This is safe choice and works for both `exim4-\*` and `postfix`.) TIP: The contents of "`/etc/mailname`" is used by many non-MTA programs for their default behavior. For `mutt`, set "`hostname`" and "`from`" variables in `\~/muttrc` file to override the **mailname** value. For programs in the `devscripts` package, such as `bts`(1) and `dch`(1), export environment variables "`$DEBFULLNAME`" and "`$DEBEMAIL`" to override it. TIP: The `popularity-contest` package normally send mail from root account with FQDN. You need to set `MAILFROM` in `/etc/popularity-contest.conf` as described in the `/usr/share/popularity-contest/default.conf` file. Otherwise, your mail will be rejected by the smarthost SMTP server. Although this is tedious, this approach is safer than rewriting the source address for all mails from root by MTA and should be used for other daemons and cron scripts. When setting the **mailname** to "`hostname -f`", the spoofing of the source mail address via MTA can be realized by the following. - "`/etc/email-addresses`" file for `exim4`(8) as explained in the `exim4-config_files`(5) - "`/etc/postfix/generic`" file for `postfix`(1) as explained in the `generic`(5) For `postfix`, the following extra steps are needed. -------------------- # postmap hash:/etc/postfix/generic # postconf -e 'smtp_generic_maps = hash:/etc/postfix/generic' # postfix reload -------------------- You can test mail address configuration using the following. - `exim`(8) with `-brw, -bf, -bF, -bV, ...` options - `postmap`(1) with `-q` option. TIP: Exim comes with several utility programs such as `exiqgrep`(8) and `exipick`(8). See "`dpkg -L exim4-base|grep man8/`" for available commands. ==== Basic MTA operations There are several basic MTA operations. Some may be performed via `sendmail`(1) compatibility interface. .List of basic MTA operation [grid="all"] `---------------------`-----------------------------------------`--------------------------------------------------------------- exim command postfix command description -------------------------------------------------------------------------------------------------------------------------------- `sendmail` `sendmail` read mails from standard input and arrange for delivery (`-bm`) `mailq` `mailq` list the mail queue with status and queue ID (`-bp`) `newaliases` `newaliases` initialize alias database (`-I`) `exim4 -q` `postqueue -f` flush waiting mails (`-q`) `exim4 -qf` `postsuper -r ALL deferred; postqueue -f` flush all mails `exim4 -qff` `postsuper -r ALL; postqueue -f` flush even frozen mails `exim4 -Mg queue_id` `postsuper -h queue_id` freeze one message by its queue ID `exim4 -Mrm queue_id` `postsuper -d queue_id` remove one message by its queue ID N/A `postsuper -d ALL` remove all messages -------------------------------------------------------------------------------------------------------------------------------- TIP: It may be a good idea to flush all mails by a script in "`/etc/ppp/ip-up.d/*`". === Mail user agent (MUA) If you subscribe to Debian related mailing list, it may be a good idea to use such MUA as `mutt` and `mew` which are the de facto standard for the participant and known to behave as expected. .List of mail user agent (MUA) [grid="all"] `------------`-------------`------------`-------------------------------------------------------------------------------------------------- package popcon size type ------------------------------------------------------------------------------------------------------------------------------------------- `evolution` @-@popcon1@-@ @-@psize1@-@ X GUI program (GNOME3, groupware suite) `icedove` @-@popcon1@-@ @-@psize1@-@ X GUI program (GNOME2, http://en.wikipedia.org/wiki/Mozilla_Corporation_software_rebranded_by_the_Debian_project[unbranded] http://en.wikipedia.org/wiki/Mozilla_Thunderbird[Mozilla Thunderbird]) `kmail` @-@popcon1@-@ @-@psize1@-@ X GUI program (KDE) `mutt` @-@popcon1@-@ @-@psize1@-@ character terminal program probably used with `vim` `mew` @-@popcon1@-@ @-@psize1@-@ character terminal program under `(x)emacs` ------------------------------------------------------------------------------------------------------------------------------------------- ==== Basic MUA — Mutt Customize "`\~/.muttrc`" as the following to use `mutt` as the mail user agent (MUA) in combination with `vim`. -------------------- # # User configuration file to override /etc/Muttrc # # spoof source mail address set use_from set hostname=example.dom set from="Name Surname " set signature="~/.signature" # vim: "gq" to reformat quotes set editor="vim -c 'set tw=72 et ft=mail'" # "mutt" goes to Inbox, while "mutt -y" lists mailboxes set mbox_type=Maildir # use qmail Maildir format for creating mbox set mbox=~/Mail # keep all mail boxes in $HOME/Mail/ set spoolfile=+Inbox # mail delivered to $HOME/Mail/Inbox set record=+Outbox # save fcc mail to $HOME/Mail/Outbox set postponed=+Postponed # keep postponed in $HOME/Mail/postponed set move=no # do not move Inbox items to mbox set quit=ask-yes # do not quit by "q" only set delete=yes # always delete w/o asking while exiting set fcc_clear # store fcc as non encrypted # Mailboxes in Maildir (automatic update) mailboxes `cd ~/Mail; /bin/ls -1|sed -e 's/^/+/' | tr "\n" " "` unmailboxes Maillog *.ev-summary ## Default #set index_format="%4C %Z %{%b %d} %-15.15L (%4l) %s" ## Thread index with senders (collapse) set index_format="%4C %Z %{%b %d} %-15.15n %?M?(#%03M)&(%4l)? %s" ## Default #set folder_format="%2C %t %N %F %2l %-8.8u %-8.8g %8s %d %f" ## just folder names set folder_format="%2C %t %N %f" -------------------- Add the following to "`/etc/mailcap`" or "`\~/.mailcap`" to display HTML mail and MS Word attachments inline. -------------------- text/html; lynx -force_html %s; needsterminal; application/msword; /usr/bin/antiword '%s'; copiousoutput; description="Microsoft Word Text"; nametemplate=%s.doc -------------------- TIP: Mutt can be used as the http://en.wikipedia.org/wiki/Internet_Message_Access_Protocol[IMAP] client and the mailbox format converter. You can tag messages with "`t`", "`T`", etc. These tagged messages can be copied with "`;C`" between different mailboxes and deleted with "`;d`" in one action. === The remote mail retrieval and forward utility Although `fetchmail`(1) has been de facto standard for the remote mail retrieval on GNU/Linux, the author likes `getmail`(1) now. If you want to reject mail before downloading to save bandwidth, `mailfilter` or `mpop` may be useful. Whichever mail retriever utilities are used, it is a good idea to configure system to deliver retrieved mails to MDA, such as `maildrop`, via pipe. .List of remote mail retrieval and forward utilities [grid="all"] `------------`-------------`------------`------------------------------------------------------------------- package popcon size description ------------------------------------------------------------------------------------------------------------- `fetchmail` @-@popcon1@-@ @-@psize1@-@ mail retriever (POP3, APOP, IMAP) (old) `getmail4` @-@popcon1@-@ @-@psize1@-@ mail retriever (POP3, IMAP4, and SDPS) (simple, secure, and reliable) `mailfilter` @-@popcon1@-@ @-@psize1@-@ mail retriever (POP3) with with regex filtering capability `mpop` @-@popcon1@-@ @-@psize1@-@ mail retriever (POP3) and MDA with filtering capability ------------------------------------------------------------------------------------------------------------- ==== getmail configuration `getmail`(1) configuration is described in http://pyropus.ca/software/getmail/documentation.html[getmail documentation]. Here is my set up to access multiple POP3 accounts as user. Create "`/usr/local/bin/getmails`" as the following. -------------------- #!/bin/sh set -e if [ -f $HOME/.getmail/running ]; then echo "getmail is already running ... (if not, remove $HOME/.getmail/running)" >&2 pgrep -l "getmai[l]" exit 1 else echo "getmail has not been running ... " >&2 fi if [ -f $HOME/.getmail/stop ]; then echo "do not run getmail ... (if not, remove $HOME/.getmail/stop)" >&2 exit fi if [ "x$1" = "x-l" ]; then exit fi rcfiles="/usr/bin/getmail" for file in $HOME/.getmail/config/* ; do rcfiles="$rcfiles --rcfile $file" done date -u > $HOME/.getmail/running eval "$rcfiles $@" rm $HOME/.getmail/running -------------------- Configure it as the following. -------------------- $ sudo chmod 755 /usr/local/bin/getmails $ mkdir -m 0700 $HOME/.getmail $ mkdir -m 0700 $HOME/.getmail/config $ mkdir -m 0700 $HOME/.getmail/log -------------------- Create configuration files "`$HOME/.getmail/config/pop3_name`" for each POP3 accounts as the following. -------------------- [retriever] type = SimplePOP3SSLRetriever server = pop.example.com username = pop3_name@example.com password = secret [destination] type = MDA_external path = /usr/bin/maildrop unixfrom = True [options] verbose = 0 delete = True delivered_to = False message_log = ~/.getmail/log/pop3_name.log -------------------- Configure it as the following. -------------------- $ chmod 0600 $HOME/.getmail/config/* -------------------- Schedule "`/usr/local/bin/getmails`" to run every 15 minutes with `cron`(8) by executing "`sudo crontab -e -u `" and adding following to user's cron entry. -------------------- 5,20,35,50 * * * * /usr/local/bin/getmails --quiet -------------------- TIP: Problems of POP3 access may not come from `getmail`. Some popular free POP3 services may be violating the POP3 protocol and their SPAM filter may not be perfect. For example, they may delete messages just after receiving RETR command before receiving DELE command and may quarantined messages into Spam mailbox. You should minimize damages by configuring them to archive accessed messages and not to delete them. See also http://mail.google.com/support/bin/answer.py?answer=13291@@@amp@@@topic=1555["Some mail was not downloaded"]. ==== fetchmail configuration `fetchmail`(1) configuration is set by "`/etc/default/fetchmail`", "`/etc/fetchmailrc`" and "`$HOME/.fetchmailrc`". See its example in "`/usr/share/doc/fetchmail/examples/fetchmailrc.example`". === Mail delivery agent (MDA) with filter Most MTA programs, such as `postfix` and `exim4`, function as MDA (mail delivery agent). There are specialized MDA with filtering capabilities. Although `procmail`(1) has been de facto standard for MDA with filter on GNU/Linux, author likes `maildrop`(1) now. Whichever filtering utilities are used, it is a good idea to configure system to deliver filtered mails to a http://en.wikipedia.org/wiki/Maildir[qmail-style Maildir]. .List of MDA with filter [grid="all"] `-----------`-------------`------------`-------------------------------------- package popcon size description ------------------------------------------------------------------------------ `procmail` @-@popcon1@-@ @-@psize1@-@ MDA with filter (old) `mailagent` @-@popcon1@-@ @-@psize1@-@ MDA with Perl filter `maildrop` @-@popcon1@-@ @-@psize1@-@ MDA with structured filtering language ------------------------------------------------------------------------------ ==== maildrop configuration `maildrop`(1) configuration is described in http://www.courier-mta.org/maildrop/maildropfilter.html[maildropfilter documentation]. Here is a configuration example for "`$HOME/.mailfilter`". -------------------- # Local configuration MAILROOT="$HOME/Mail" # set this to /etc/mailname contents MAILHOST="example.dom" logfile $HOME/.maildroplog # rules are made to override the earlier value by the later one. # mailing list mails ? if ( /^Precedence:.*list/:h || /^Precedence:.*bulk/:h ) { # rules for mailing list mails # default mailbox for mails from mailing list MAILBOX="Inbox-list" # default mailbox for mails from debian.org if ( /^(Sender|Resent-From|Resent-Sender): .*debian.org/:h ) { MAILBOX="service.debian.org" } # default mailbox for mails from bugs.debian.org (BTS) if ( /^(Sender|Resent-From|Resent-sender): .*@bugs.debian.org/:h ) { MAILBOX="bugs.debian.org" } # mailbox for each properly maintained mailing list with "List-Id: foo" or "List-Id: ..." if ( /^List-Id: ([^<]*<)?([^<>]*)>?/:h ) { MAILBOX="$MATCH2" } } else { # rules for non-mailing list mails # default incoming box MAILBOX="Inbox-unusual" # local mails if ( /Envelope-to: .*@$MAILHOST/:h ) { MAILBOX="Inbox-local" } # html mails (99% spams) if ( /DOCTYPE html/:b ||\ /^Content-Type: text\/html/ ) { MAILBOX="Inbox-html" } # blacklist rule for spams if ( /^X-Advertisement/:h ||\ /^Subject:.*BUSINESS PROPOSAL/:h ||\ /^Subject:.*URGENT.*ASISSTANCE/:h ||\ /^Subject: *I NEED YOUR ASSISTANCE/:h ) { MAILBOX="Inbox-trash" } # whitelist rule for normal mails if ( /^From: .*@debian.org/:h ||\ /^(Sender|Resent-From|Resent-Sender): .*debian.org/:h ||\ /^Subject: .*(debian|bug|PATCH)/:h ) { MAILBOX="Inbox" } # whiltelist rule for BTS related mails if ( /^Subject: .*Bug#.*/:h ||\ /^(To|Cc): .*@bugs.debian.org/:h ) { MAILBOX="bugs.debian.org" } # whitelist rule for getmails cron mails if ( /^Subject: Cron .*getmails/:h ) { MAILBOX="Inbox-getmails" } } # check existance of $MAILBOX `test -d $MAILROOT/$MAILBOX` if ( $RETURNCODE == 1 ) { # create maildir mailbox for $MAILBOX `maildirmake $MAILROOT/$MAILBOX` } # deliver to maildir $MAILBOX to "$MAILROOT/$MAILBOX/" exit -------------------- WARNING: Unlike `procmail`, `maildrop` does not create missing maildir directories automatically. You must create them manually using `maildirmake`(1) in advance as in the example "`$HOME/.mailfilter`". ==== procmail configuration Here is a similar configuration with "`$HOME/.procmailrc`" for `procmail`(1). -------------------- MAILDIR=$HOME/Maildir DEFAULT=$MAILDIR/Inbox/ LOGFILE=$MAILDIR/Maillog # clearly bad looking mails: drop them into X-trash and exit :0 * 1^0 ^X-Advertisement * 1^0 ^Subject:.*BUSINESS PROPOSAL * 1^0 ^Subject:.*URGENT.*ASISSTANCE * 1^0 ^Subject: *I NEED YOUR ASSISTANCE X-trash/ # Delivering mailinglist messages :0 * 1^0 ^Precedence:.*list * 1^0 ^Precedence:.*bulk * 1^0 ^List- * 1^0 ^X-Distribution:.*bulk { :0 * 1^0 ^Return-path:.*debian-devel-admin@debian.or.jp jp-debian-devel/ :0 * ^Resent-Sender.*debian-user-request@lists.debian.org debian-user/ :0 * ^Resent-Sender.*debian-devel-request@lists.debian.org debian-devel/ :0 * ^Resent-Sender.*debian-announce-request@lists.debian.org debian-announce :0 mailing-list/ } :0 Inbox/ -------------------- ==== Redeliver mbox contents You need to manually deliver mails to the sorted mailboxes in your home directory from "`/var/mail/`" if your home directory became full and `procmail`(1) failed. After making disk space in the home directory, run the following. -------------------- # /etc/init.d/${MAILDAEMON} stop # formail -s procmail # /etc/init.d/${MAILDAEMON} start -------------------- === POP3/IMAP4 server If you are to run a private server on LAN, you may consider to run http://en.wikipedia.org/wiki/Post_Office_Protocol[POP3] / http://en.wikipedia.org/wiki/Internet_Message_Access_Protocol[IMAP4] server for delivering mail to LAN clients. .List of POP3/IMAP4 servers [grid="all"] `-----------------`-------------`------------`----`---------------------------------------------------------- package popcon size type description ------------------------------------------------------------------------------------------------------------- `courier-pop` @-@popcon1@-@ @-@psize1@-@ POP3 Courier mail server - POP3 server (maildir format only) `cyrus-pop3d` @-@popcon1@-@ @-@psize1@-@ POP3 Cyrus mail system (POP3 support) `xmail` @-@popcon1@-@ @-@psize1@-@ POP3 ESMTP/POP3 mail server `courier-imap` @-@popcon1@-@ @-@psize1@-@ IMAP Courier mail server - IMAP server (maildir format only) `cyrus-imapd` @-@popcon1@-@ @-@psize1@-@ IMAP Cyrus mail system (IMAP support) ------------------------------------------------------------------------------------------------------------- === The print server and utility In the old Unix-like system, the BSD http://en.wikipedia.org/wiki/Line_Printer_Daemon_protocol[Line printer daemon] was the standard. Since the standard print out format of the free software is PostScript on the Unix like system, some filter system was used along with http://en.wikipedia.org/wiki/Ghostscript[Ghostscript] to enable printing to the non-PostScript printer. Recently, http://en.wikipedia.org/wiki/Common_Unix_Printing_System[Common UNIX Printing System] (CUPS) is the new de facto standard. The CUPS uses http://en.wikipedia.org/wiki/Internet_Printing_Protocol[Internet Printing Protocol] (IPP). The IPP is now supported by other OSs such as Windows XP and Mac OS X and has became new cross-platform de facto standard for remote printing with bi-directional communication capability. The standard printable data format for the application on the Debian system is the http://en.wikipedia.org/wiki/PostScript[PostScript (PS)] which is a page description language. The data in PS format is fed into the Ghostscript PostScript interpreter to produce the printable data specific to the printer. See <<_ghostscript>>. Thanks to the file format dependent auto-conversion feature of the CUPS system, simply feeding any data to the `lpr` command should generate the expected print output. (In CUPS, `lpr` can be enabled by installing the `cups-bsd` package.) The Debian system has some notable packages for the print servers and utilities. .List of print servers and utilities [grid="all"] `------------------------`-------------`------------`--------------`---------------------------------------------------------------------- package popcon size port description ------------------------------------------------------------------------------------------------------------------------------------------ `lpr` @-@popcon1@-@ @-@psize1@-@ printer (515) BSD lpr/lpd (http://en.wikipedia.org/wiki/Line_Printer_Daemon_protocol[Line printer daemon]) `lprng` @-@popcon1@-@ @-@psize1@-@ , , , , (Enhanced) `cups` @-@popcon1@-@ @-@psize1@-@ IPP (631) Internet Printing CUPS server `cups-client` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/System_V_printing_system[System V printer commands] for CUPS: `lp`(1), `lpstat`(1), `lpoptions`(1), `cancel`(1), `lpmove`(8), `lpinfo`(8), `lpadmin`(8), ... `cups-bsd` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/Line_Printer_Daemon_protocol[BSD printer commands] for CUPS: `lpr`(1), `lpq`(1), `lprm`(1), `lpc`(8) `cups-driver-gutenprint` @-@popcon1@-@ @-@psize1@-@ Not applicable printer drivers for CUPS ------------------------------------------------------------------------------------------------------------------------------------------ TIP: You can configure CUPS system by pointing your web browser to "http://localhost:631/[http://localhost:631/]" . === The remote access server and utility (SSH) The http://en.wikipedia.org/wiki/Secure_Shell[Secure SHell] (SSH) is the **secure** way to connect over the Internet. A free version of SSH called http://www.openssh.org/[OpenSSH] is available as `openssh-client` and `openssh-server` packages in Debian. .List of remote access server and utilities [grid="all"] `------------------------`-------------`------------`---------------------------`------------------------------------------------- package popcon size tool description ---------------------------------------------------------------------------------------------------------------------------------- `openssh-client` @-@popcon1@-@ @-@psize1@-@ `ssh`(1) Secure shell client `openssh-server` @-@popcon1@-@ @-@psize1@-@ `sshd`(8) Secure shell server `ssh-askpass-fullscreen` @-@popcon1@-@ @-@psize1@-@ `ssh-askpass-fullscreen`(1) asks user for a pass phrase for ssh-add (GNOME2) `ssh-askpass` @-@popcon1@-@ @-@psize1@-@ `ssh-askpass`(1) asks user for a pass phrase for ssh-add (plain X) ---------------------------------------------------------------------------------------------------------------------------------- CAUTION: See <<_extra_security_measures_for_the_internet>> if your SSH is accessible from the Internet. TIP: Please use the `screen`(1) program to enable remote shell process to survive the interrupted connection (see <<_the_screen_program>>). ==== Basics of SSH WARNING: "`/etc/ssh/sshd_not_to_be_run`" must not be present if one wishes to run the OpenSSH server. SSH has two authentication protocols. .List of SSH authentication protocols and methods [grid="all"] `------------`-----------------------------------`---------------------------------------------------------------------------------------- SSH protocol SSH method description ------------------------------------------------------------------------------------------------------------------------------------------ SSH-1 "`RSAAuthentication`" RSA identity key based user authentication , , "`RhostsAuthentication`" "`.rhosts`" based host authentication (insecure, disabled) , , "`RhostsRSAAuthentication`" "`.rhosts`" based host authentication combined with RSA host key (disabled) , , "`ChallengeResponseAuthentication`" RSA challenge-response authentication , , "`PasswordAuthentication`" password based authentication SSH-2 "`PubkeyAuthentication`" public key based user authentication , , "`HostbasedAuthentication`" "`\~/.rhosts`" or "`/etc/hosts.equiv`" based host authentication combined with public key client host authentication (disabled) , , "`ChallengeResponseAuthentication`" challenge-response authentication , , "`PasswordAuthentication`" password based authentication ------------------------------------------------------------------------------------------------------------------------------------------ CAUTION: Be careful about these differences if you are using a non-Debian system. See "`/usr/share/doc/ssh/README.Debian.gz`", `ssh`(1), `sshd`(8), `ssh-agent`(1), and `ssh-keygen`(1) for details. Following are the key configuration files. .List of SSH configuration files [grid="all"] `-------------------------`-------------------------------------------------------------------------------------------------- configuration file description of configuration file ----------------------------------------------------------------------------------------------------------------------------- `/etc/ssh/ssh_config` SSH client defaults, see `ssh_config`(5) `/etc/ssh/sshd_config` SSH server defaults, see `sshd_config`(5) `\~/.ssh/authorized_keys` default public SSH keys that clients use to connect to this account on this SSH server `\~/.ssh/identity` secret SSH-1 RSA key of the user `\~/.ssh/id_rsa` secret SSH-2 RSA key of the user `\~/.ssh/id_dsa` secret SSH-2 DSA key of the user ----------------------------------------------------------------------------------------------------------------------------- TIP: See `ssh-keygen`(1), `ssh-add`(1) and `ssh-agent`(1) for how to use public and secret SSH keys. TIP: Make sure to verify settings by testing the connection. In case of any problem, use "`ssh -v`". TIP: You can change the pass phrase to encrypt local secret SSH keys later with "`ssh-keygen -p`". TIP: You can add options to the entries in "`\~/.ssh/authorized_keys`" to limit hosts and to run specific commands. See `sshd`(8) for details. The following starts an `ssh`(1) connection from a client. .List of SSH client startup examples [grid="all"] `--------------------------------------------------------------------------`------------------------------------------------- command description ----------------------------------------------------------------------------------------------------------------------------- `ssh username@hostname.domain.ext` connect with default mode `ssh -v username@hostname.domain.ext` connect with default mode with debugging messages `ssh -1 username@hostname.domain.ext` force to connect with SSH version 1 `ssh -1 -o RSAAuthentication=no -l username hostname.domain.ext` force to use password with SSH version 1 `ssh -o PreferredAuthentications=password -l username hostname.domain.ext` force to use password with SSH version 2 ----------------------------------------------------------------------------------------------------------------------------- If you use the same user name on the local and the remote host, you can eliminate typing "`username@`". Even if you use different user name on the local and the remote host, you can eliminate it using "`\~/.ssh/config`". For http://alioth.debian.org/[Debian Alioth service] with account name "`foo-guest`", you set "`\~/.ssh/config`" to contain the following. -------------------- Host alioth.debian.org svn.debian.org git.debian.org User foo-guest -------------------- For the user, `ssh`(1) functions as a smarter and more secure `telnet`(1). Unlike `telnet` command, `ssh` command does not bomb on the `telnet` escape character (initial default CTRL-]). ==== Port forwarding for SMTP/POP3 tunneling To establish a pipe to connect to port 25 of `remote-server` from port 4025 of `localhost`, and to port 110 of `remote-server` from port 4110 of `localhost` through `ssh`, execute on the local host as the following. -------------------- # ssh -q -L 4025:remote-server:25 4110:remote-server:110 username@remote-server -------------------- This is a secure way to make connections to SMTP/POP3 servers over the Internet. Set the "`AllowTcpForwarding`" entry to "`yes`" in "`/etc/ssh/sshd_config`" of the remote host. ==== Connecting without remote passwords One can avoid having to remember passwords for remote systems by using "`RSAAuthentication`" (SSH-1 protocol) or "`PubkeyAuthentication`" (SSH-2 protocol). On the remote system, set the respective entries, "`RSAAuthentication yes`" or "`PubkeyAuthentication yes`", in "`/etc/ssh/sshd_config`". Generate authentication keys locally and install the public key on the remote system by the following. - "`RSAAuthentication`": RSA key for SSH-1 (deprecated because it is superseded.) -------------------- $ ssh-keygen $ cat .ssh/identity.pub | ssh user1@remote "cat - >>.ssh/authorized_keys" -------------------- - "`PubkeyAuthentication`": RSA key for SSH-2 -------------------- $ ssh-keygen -t rsa $ cat .ssh/id_rsa.pub | ssh user1@remote "cat - >>.ssh/authorized_keys" -------------------- - "`PubkeyAuthentication`": DSA key for SSH-2 (deprecated because it is slow.) -------------------- $ ssh-keygen -t dsa $ cat .ssh/id_dsa.pub | ssh user1@remote "cat - >>.ssh/authorized_keys" -------------------- TIP: Use of DSA key for SSH-2 is deprecated because key is smaller and slow. There are no more reasons to work around RSA patent using DSA since it has been expired. DSA stands for http://en.wikipedia.org/wiki/Digital_Signature_Algorithm[Digital Signature Algorithm] and slow. Also see http://www.debian.org/security/2008/dsa-1571[DSA-1571-1]. NOTE: For "`HostbasedAuthentication`" to work in SSH-2, you must adjust the settings of "`HostbasedAuthentication`" to "`yes`" in both "`/etc/ssh/sshd_config`" on the server host and "`/etc/ssh/ssh_config`" or "`\~/.ssh/config`" on the client host. ==== Dealing with alien SSH clients There are some free http://en.wikipedia.org/wiki/Secure_Shell[SSH] clients available for other platforms. .List of free SSH clients for other platforms [grid="all"] `-----------------`----------------------------------------------------------------- environment free SSH program ------------------------------------------------------------------------------------ Windows puTTY (http://www.chiark.greenend.org.uk/\~sgtatham/putty/) (GPL) Windows (cygwin) SSH in cygwin (http://www.cygwin.com/) (GPL) Macintosh Classic macSSH (http://www.macssh.com/) (GPL) Mac OS X OpenSSH; use `ssh` in the Terminal application (GPL) ------------------------------------------------------------------------------------ ==== Setting up ssh-agent It is safer to protect your SSH authentication secret keys with a pass phrase. If a pass phrase was not set, use "`ssh-keygen -p`" to set it. Place your public SSH key (e.g. "`\~/.ssh/id_rsa.pub`") into "`\~/.ssh/authorized_keys`" on a remote host using a password-based connection to the remote host as described above. -------------------- $ ssh-agent bash $ ssh-add ~/.ssh/id_rsa Enter passphrase for /home//.ssh/id_rsa: Identity added: /home//.ssh/id_rsa (/home//.ssh/id_rsa) -------------------- No remote password needed from here on for the next command. -------------------- $ scp foo @remote.host:foo -------------------- Press \^D to terminating ssh-agent session. For the X server, the normal Debian startup script executes `ssh-agent` as the parent process. So you only need to execute `ssh-add` once. For more, read `ssh-agent`(1)and `ssh-add`(1). ==== How to shutdown the remote system on SSH You need to protect the process doing "`shutdown -h now`" (see <<_how_to_shutdown_the_system>>) from the termination of SSH using the `at`(1) command (see <<_scheduling_tasks_once>>) by the following. -------------------- # echo "shutdown -h now" | at now -------------------- Running "`shutdown -h now`" in `screen`(1) (see <<_the_screen_program>>) session is another way to do the same. ==== Troubleshooting SSH If you have problems, check the permissions of configuration files and run `ssh` with the "`-v`" option. Use the "`-P`" option if you are root and have trouble with a firewall; this avoids the use of server ports 1 -- 1023. If `ssh` connections to a remote site suddenly stop working, it may be the result of tinkering by the sysadmin, most likely a change in "`host_key`" during system maintenance. After making sure this is the case and nobody is trying to fake the remote host by some clever hack, one can regain a connection by removing the "`host_key`" entry from "`\~/.ssh/known_hosts`" on the local host. === Other network application servers Here are other network application servers. .List of other network application servers [grid="all"] `---------------------`-------------`------------`----------------------------------------------------------------------`-------------------------------------------------------------------------------------------------------------------------- package popcon size protocol description --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- `telnetd` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/TELNET[TELNET] TELNET server `telnetd-ssl` @-@popcon1@-@ @-@psize1@-@ , , , , (SSL support) `nfs-kernel-server` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Network_File_System_(protocol)[NFS] Unix file sharing `samba` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Server_Message_Block[SMB] Windows file and printer sharing `netatalk` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/AppleTalk[ATP] Apple/Mac file and printer sharing (AppleTalk) `proftpd-basic` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/File_Transfer_Protocol[FTP] General file download `apache2-mpm-prefork` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol[HTTP] General web server `apache2-mpm-worker` @-@popcon1@-@ @-@psize1@-@ , , , , `squid` @-@popcon1@-@ @-@psize1@-@ , , General web http://en.wikipedia.org/wiki/Proxy_server[proxy server] `squid3` @-@popcon1@-@ @-@psize1@-@ , , , , `slpd` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Service_Location_Protocol[SLP] http://www.openslp.org/[OpenSLP] Server as http://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol[LDAP] server `bind9` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Domain_Name_System[DNS] IP address for other hosts `isc-dhcp-server` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol[DHCP] IP address of client itself --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Common Internet File System Protocol (CIFS) is the same protocol as http://en.wikipedia.org/wiki/Server_Message_Block[Server Message Block (SMB)] and is used widely by Microsoft Windows. TIP: See <<_the_modern_centralized_system_management>> for integration of server systems. TIP: The hostname resolution is usually provided by the http://en.wikipedia.org/wiki/Domain_Name_System[DNS] server. For the host IP address dynamically assigned by http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol[DHCP], http://en.wikipedia.org/wiki/Dynamic_DNS[Dynamic DNS] can be set up for the hostname resolution using `bind9` and `isc-dhcp-server` as described in the http://wiki.debian.org/DDNS[DDNS page on the Debian wiki]. TIP: Use of proxy server such as `squid` is much more efficient for saving bandwidth than use of local mirror server with the full Debian archive contents. === Other network application clients Here are other network application clients. .List of network application clients [grid="all"] `-----------------`-------------`------------`---------------------------------------------------------------------------------------------------------------------------`---------------------- package popcon size protocol description ------------------------------------------------------------------------------------------------------------------------------------------ `netcat` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/TCP/IP[TCP/IP] TCP/IP swiss army knife `openssl` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Transport_Layer_Security[SSL] Secure Socket Layer (SSL) binary and related cryptographic tools `stunnel4` @-@popcon1@-@ @-@psize1@-@ , , universal SSL Wrapper `telnet` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/TELNET[TELNET] TELNET client `telnet-ssl` @-@popcon1@-@ @-@psize1@-@ , , , , (SSL support) `nfs-common` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Network_File_System_(protocol)[NFS] Unix file sharing `smbclient` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Server_Message_Block[SMB] MS Windows file and printer sharing client `cifs-utils` @-@popcon1@-@ @-@psize1@-@ , , mount and umount commands for remote MS Windows file `ftp` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/File_Transfer_Protocol[FTP] FTP client `lftp` @-@popcon1@-@ @-@psize1@-@ , , , , `ncftp` @-@popcon1@-@ @-@psize1@-@ , , full screen FTP client `wget` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol[HTTP] and http://en.wikipedia.org/wiki/File_Transfer_Protocol[FTP] web downloader `curl` @-@popcon1@-@ @-@psize1@-@ , , , , `axel` @-@popcon1@-@ @-@psize1@-@ , , accelerated downloader `aria2` @-@popcon1@-@ @-@psize1@-@ , , accelerated downloader with http://en.wikipedia.org/wiki/BitTorrent_(protocol)[BitTorrent] and http://en.wikipedia.org/wiki/Metalink[Metalink] supports `bind9-host` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Domain_Name_System[DNS] `host`(1) from bind9, "`Priority: standard`" `dnsutils` @-@popcon1@-@ @-@psize1@-@ , , `dig`(1) from bind, "`Priority: standard`" `isc-dhcp-client` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol[DHCP] obtain IP address `ldap-utils` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol[LDAP] obtain data from LDAP server ------------------------------------------------------------------------------------------------------------------------------------------ === The diagnosis of the system daemons The `telnet` program enables manual connection to the system daemons and its diagnosis. For testing plain http://en.wikipedia.org/wiki/Post_Office_Protocol[POP3] service, try the following -------------------- $ telnet mail.ispname.net pop3 -------------------- For testing the http://en.wikipedia.org/wiki/Transport_Layer_Security[TLS]/SSL enabled http://en.wikipedia.org/wiki/Post_Office_Protocol[POP3] service by some ISPs, you need TLS/SSL enabled `telnet` client by the `telnet-ssl` or `openssl` packages. -------------------- $ telnet -z ssl pop.gmail.com 995 -------------------- -------------------- $ openssl s_client -connect pop.gmail.com:995 -------------------- The following http://www.ietf.org/rfc.html[RFCs] provide required knowledge to each system daemon. .List of popular RFCs [grid="all"] `-------------------------------------------------------------------------------------------`--------------------------------------------- RFC description ------------------------------------------------------------------------------------------------------------------------------------------ http://tools.ietf.org/html/rfc1939[rfc1939] and http://tools.ietf.org/html/rfc2449[rfc2449] http://en.wikipedia.org/wiki/Post_Office_Protocol[POP3] service http://tools.ietf.org/html/rfc3501[rfc3501] http://en.wikipedia.org/wiki/Internet_Message_Access_Protocol[IMAP4] service http://tools.ietf.org/html/rfc2821[rfc2821] (http://tools.ietf.org/html/rfc821[rfc821]) http://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol[SMTP] service http://tools.ietf.org/html/rfc2822[rfc2822] (http://tools.ietf.org/html/rfc822[rfc822]) Mail file format http://tools.ietf.org/html/rfc2045[rfc2045] http://en.wikipedia.org/wiki/MIME[Multipurpose Internet Mail Extensions (MIME)] http://tools.ietf.org/html/rfc819[rfc819] http://en.wikipedia.org/wiki/Domain_Name_System[DNS] service http://tools.ietf.org/html/rfc2616[rfc2616] http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol[HTTP] service http://tools.ietf.org/html/rfc2396[rfc2396] http://en.wikipedia.org/wiki/Uniform_Resource_Identifier[URI] definition ------------------------------------------------------------------------------------------------------------------------------------------ The port usage is described in "`/etc/services`". debian-reference-2.53/asciidoc/header1.txt0000644000000000000000000000146412255257602015352 0ustar %dbcent; %commondata; %popcon; %datadate1; %pkgsize; %datadate2; debian-reference-2.53/asciidoc/10_datamngt.txt0000644000000000000000000023622512255345173016146 0ustar == Data management // vim: set sts=2 expandtab: // Use ":set nowrap" to edit table Tools and tips for managing binary and text data on the Debian system are described. WARNING: The uncoordinated write access to actively accessed devices and files from multiple processes must not be done to avoid the http://en.wikipedia.org/wiki/Race_condition[race condition]. http://en.wikipedia.org/wiki/File_locking[File locking] mechanisms using `flock`(1) may be used to avoid it. === Sharing, copying, and archiving The security of the data and its controlled sharing have several aspects. - The creation of data archive - The remote storage access - The duplication - The tracking of the modification history - The facilitation of data sharing - The prevention of unauthorized file access - The detection of unauthorized file modification These can be realized by using some combination of tools. - Archive and compression tools - Copy and synchronization tools - Network filesystems - Removable storage media - The secure shell - The authentication system - Version control system tools - Hash and cryptographic encryption tools ==== Archive and compression tools Here is a summary of archive and compression tools available on the Debian system. .List of archive and compression tools [grid="all"] `------------`-------------`------------`---------`---------------------------`----------------------------------------------------------- package popcon size extension command comment ------------------------------------------------------------------------------------------------------------------------------------------ `tar` @-@popcon1@-@ @-@psize1@-@ `.tar` `tar`(1) the standard archiver (de facto standard) `cpio` @-@popcon1@-@ @-@psize1@-@ `.cpio` `cpio`(1) Unix System V style archiver, use with `find`(1) `binutils` @-@popcon1@-@ @-@psize1@-@ `.ar` `ar`(1) archiver for the creation of static libraries `fastjar` @-@popcon1@-@ @-@psize1@-@ `.jar` `fastjar`(1) archiver for Java (zip like) `pax` @-@popcon1@-@ @-@psize1@-@ `.pax` `pax`(1) new POSIX standard archiver, compromise between `tar` and `cpio` `gzip` @-@popcon1@-@ @-@psize1@-@ `.gz` `gzip`(1), `zcat`(1), ... GNU http://en.wikipedia.org/wiki/LZ77_and_LZ78[LZ77] compression utility (de facto standard) `bzip2` @-@popcon1@-@ @-@psize1@-@ `.bz2` `bzip2`(1), `bzcat`(1), ... http://en.wikipedia.org/wiki/Burrows-Wheeler_transform[Burrows-Wheeler block-sorting compression] utility with higher compression ratio than `gzip`(1) (slower than `gzip` with similar syntax) `lzma` @-@popcon1@-@ @-@psize1@-@ `.lzma` `lzma`(1) http://en.wikipedia.org/wiki/Lempel-Ziv-Markov_chain_algorithm[LZMA] compression utility with higher compression ratio than `gzip`(1) (deprecated) `xz-utils` @-@popcon1@-@ @-@psize1@-@ `.xz` `xz`(1), `xzdec`(1), ... http://en.wikipedia.org/wiki/Xz[XZ] compression utility with higher compression ratio than `bzip2`(1) (slower than `gzip` but faster than `bzip2`; replacement for http://en.wikipedia.org/wiki/Lempel-Ziv-Markov_chain_algorithm[LZMA] compression utility) `p7zip` @-@popcon1@-@ @-@psize1@-@ `.7z` `7zr`(1), `p7zip`(1) http://en.wikipedia.org/wiki/7-Zip[7-Zip] file archiver with high compression ratio (http://en.wikipedia.org/wiki/Lempel-Ziv-Markov_chain_algorithm[LZMA] compression) `p7zip-full` @-@popcon1@-@ @-@psize1@-@ `.7z` `7z`(1), `7za`(1) http://en.wikipedia.org/wiki/7-Zip[7-Zip] file archiver with high compression ratio (http://en.wikipedia.org/wiki/Lempel-Ziv-Markov_chain_algorithm[LZMA] compression and others) `lzop` @-@popcon1@-@ @-@psize1@-@ `.lzo` `lzop`(1) http://en.wikipedia.org/wiki/Lempel-Ziv-Oberhumer[LZO] compression utility with higher compression and decompression speed than `gzip`(1) (lower compression ratio than `gzip` with similar syntax) `zip` @-@popcon1@-@ @-@psize1@-@ `.zip` `zip`(1) http://en.wikipedia.org/wiki/Info-ZIP[InfoZIP]: DOS archive and compression tool `unzip` @-@popcon1@-@ @-@psize1@-@ `.zip` `unzip`(1) http://en.wikipedia.org/wiki/Info-ZIP[InfoZIP]: DOS unarchive and decompression tool ------------------------------------------------------------------------------------------------------------------------------------------ WARNING: Do not set the "`$TAPE`" variable unless you know what to expect. It changes `tar`(1) behavior. NOTE: The gzipped `tar`(1) archive uses the file extension "`.tgz`" or "`.tar.gz`". NOTE: The xz-compressed `tar`(1) archive uses the file extension "`.txz`" or "`.tar.xz`". NOTE: Popular compression method in http://en.wikipedia.org/wiki/Free_and_open_source_software[FOSS] tools such as `tar`(1) has been moving as follows: `gzip` -> `bzip2` -> `xz` NOTE: `cp`(1), `scp`(1) and `tar`(1) may have some limitation for special files. `cpio`(1) is most versatile. NOTE: `cpio`(1) is designed to be used with `find`(1) and other commands and suitable for creating backup scripts since the file selection part of the script can be tested independently. NOTE: Internal structure of OpenOffice data files are "`.jar`" file. ==== Copy and synchronization tools Here is a summary of simple copy and backup tools available on the Debian system. .List of copy and synchronization tools [grid="all"] `----------------`-------------`------------`------`------------------------------------------------------------------- package popcon size tool function ----------------------------------------------------------------------------------------------------------------------- `coreutils` @-@popcon1@-@ @-@psize1@-@ GNU cp locally copy files and directories ("-a" for recursive) `openssh-client` @-@popcon1@-@ @-@psize1@-@ scp remotely copy files and directories (client, "`-r`" for recursive) `openssh-server` @-@popcon1@-@ @-@psize1@-@ sshd remotely copy files and directories (remote server) `rsync` @-@popcon1@-@ @-@psize1@-@ - 1-way remote synchronization and backup `unison` @-@popcon1@-@ @-@psize1@-@ - 2-way remote synchronization and backup `obnam` @-@popcon1@-@ @-@psize1@-@ - (remote) incremental backup `rdiff-backup` @-@popcon1@-@ @-@psize1@-@ - (remote) incremental backup ----------------------------------------------------------------------------------------------------------------------- Copying files with `rsync`(8) offers richer features than others. - delta-transfer algorithm that sends only the differences between the source files and the existing files in the destination - quick check algorithm (by default) that looks for files that have changed in size or in last-modified time - "`--exclude`" and "`--exclude-from`" options similar to `tar`(1) - "a trailing slash on the source directory" syntax that avoids creating an additional directory level at the destination. TIP: Execution of the `bkup` script mentioned in <<_a_copy_script_for_the_data_backup>> with the "`-gl`" option under `cron`(8) should provide very similar functionality as Plan9's `dumpfs` for the static data archive. TIP: Version control system (VCS) tools in <> can function as the multi-way copy and synchronization tools. ==== Idioms for the archive Here are several ways to archive and unarchive the entire content of the directory "`./source`" using different tools. GNU `tar`(1): -------------------- $ tar -cvJf archive.tar.xz ./source $ tar -xvJf archive.tar.xz -------------------- Alternatively, by the following. -------------------- $ find ./source -xdev -print0 | tar -cvJf archive.tar.xz --null -F - -------------------- `cpio`(1): -------------------- $ find ./source -xdev -print0 | cpio -ov --null > archive.cpio; xz archive.cpio $ zcat archive.cpio.xz | cpio -i -------------------- ==== Idioms for the copy Here are several ways to copy the entire content of the directory "`./source`" using different tools. - Local copy: "`./source`" directory -> "`/dest`" directory - Remote copy: "`./source`" directory at local host -> "`/dest`" directory at "`user@host.dom`" host `rsync`(8): -------------------- # cd ./source; rsync -aHAXSv . /dest # cd ./source; rsync -aHAXSv . user@host.dom:/dest -------------------- You can alternatively use "a trailing slash on the source directory" syntax. -------------------- # rsync -aHAXSv ./source/ /dest # rsync -aHAXSv ./source/ user@host.dom:/dest -------------------- Alternatively, by the following. -------------------- # cd ./source; find . -print0 | rsync -aHAXSv0 --files-from=- . /dest # cd ./source; find . -print0 | rsync -aHAXSv0 --files-from=- . user@host.dom:/dest -------------------- GNU `cp`(1) and openSSH `scp`(1): -------------------- # cd ./source; cp -a . /dest # cd ./source; scp -pr . user@host.dom:/dest -------------------- GNU `tar`(1): -------------------- # (cd ./source && tar cf - . ) | (cd /dest && tar xvfp - ) # (cd ./source && tar cf - . ) | ssh user@host.dom '(cd /dest && tar xvfp - )' -------------------- `cpio`(1): -------------------- # cd ./source; find . -print0 | cpio -pvdm --null --sparse /dest -------------------- You can substitute "`.`" with "`foo`" for all examples containing "`.`" to copy files from "`./source/foo`" directory to "`/dest/foo`" directory. You can substitute "`.`" with the absolute path "`/path/to/source/foo`" for all examples containing "`.`" to drop "`cd ./source;`". These copy files to different locations depending on tools used as follows. - "`/dest/foo`": `rsync`(8), GNU `cp`(1), and `scp`(1) - "`/dest/path/to/source/foo`": GNU `tar`(1), and `cpio`(1) TIP: `rsync`(8) and GNU `cp`(1) have option "`-u`" to skip files that are newer on the receiver. ==== Idioms for the selection of files `find`(1) is used to select files for archive and copy commands (see <<_idioms_for_the_archive>> and <<_idioms_for_the_copy>>) or for `xargs`(1) (see <<_repeating_a_command_looping_over_files>>). This can be enhanced by using its command arguments. Basic syntax of `find`(1) can be summarized as the following. - Its conditional arguments are evaluated from left to right. - This evaluation stops once its outcome is determined. - "Logical **OR**" (specified by "`-o`" between conditionals) has lower precedence than "logical **AND**" (specified by "`-a`" or nothing between conditionals). - "Logical **NOT**" (specified by "`!`" before a conditional) has higher precedence than "logical **AND**". - "`-prune`" always returns logical **TRUE** and, if it is a directory, searching of file is stopped beyond this point. - "`-name`" matches the base of the filename with shell glob (see <<_shell_glob>>) but it also matches its initial "`.`" with metacharacters such as "`\*`" and "`?`". (New http://en.wikipedia.org/wiki/POSIX[POSIX] feature) - "`-regex`" matches the full path with emacs style **BRE** (see <<_regular_expressions>>) as default. - "`-size`" matches the file based on the file size (value precedented with "`+`" for larger, precedented with "`-`" for smaller) - "`-newer`" matches the file newer than the one specified in its argument. - "`-print0`" always returns logical **TRUE** and print the full filename (http://en.wikipedia.org/wiki/Null_character[null terminated]) on the standard output. `find`(1) is often used with an idiomatic style as the following. -------------------- # find /path/to \ -xdev -regextype posix-extended \ -type f -regex ".*\.cpio|.*~" -prune -o \ -type d -regex ".*/\.git" -prune -o \ -type f -size +99M -prune -o \ -type f -newer /path/to/timestamp -print0 -------------------- This means to do following actions. 1. Search all files starting from "`/path/to`" 2. Globally limit its search within its starting filesystem and uses **ERE** (see <<_regular_expressions>>) instead 3. Exclude files matching regex of "`.\*\.cpio`" or "`.\*\~`" from search by stop processing 4. Exclude directories matching regex of "`.\*/\.git`" from search by stop processing 5. Exclude files larger than 99 Megabytes (units of 1048576 bytes) from search by stop processing 6. Print filenames which satisfy above search conditions and newer than "`/path/to/timestamp`" Please note the idiomatic use of "`-prune -o`" to exclude files in the above example. NOTE: For non-Debian http://en.wikipedia.org/wiki/Unix-like[Unix-like] system, some options may not be supported by `find`(1). In such a case, please consider to adjust matching methods and replace "`-print0`" with "`-print`". You may need to adjust related commands too. ==== Archive media When choosing http://en.wikipedia.org/wiki/Computer_data_storage[computer data storage media] for important data archive, you should be careful about their limitations. For small personal data backup, I use CD-R and DVD-R by the brand name company and store in a cool, shaded, dry, clean environment. (Tape archive media seem to be popular for professional use.) NOTE: http://en.wikipedia.org/wiki/Safe[A fire-resistant safe] are meant for paper documents. Most of the computer data storage media have less temperature tolerance than paper. I usually rely on multiple secure encrypted copies stored in multiple secure locations. Optimistic storage life of archive media seen on the net (mostly from vendor info). - 100+ years : Acid free paper with ink - 100 years : Optical storage (CD/DVD, CD/DVD-R) - 30 years : Magnetic storage (tape, floppy) - 20 years : Phase change optical storage (CD-RW) These do not count on the mechanical failures due to handling etc. Optimistic write cycle of archive media seen on the net (mostly from vendor info). - 250,000+ cycles : Harddisk drive - 10,000+ cycles : Flash memory - 1,000 cycles : CD/DVD-RW - 1 cycles : CD/DVD-R, paper CAUTION: Figures of storage life and write cycle here should not be used for decisions on any critical data storage. Please consult the specific product information provided by the manufacture. TIP: Since CD/DVD-R and paper have only 1 write cycle, they inherently prevent accidental data loss by overwriting. This is advantage! TIP: If you need fast and frequent backup of large amount of data, a hard disk on a remote host linked by a fast network connection, may be the only realistic option. ==== Removable storage device Removable storage devices may be any one of the following. - http://en.wikipedia.org/wiki/USB_flash_drive[USB flash drive] - http://en.wikipedia.org/wiki/Hard_disk_drive[Hard disk drive] - http://en.wikipedia.org/wiki/Optical_disc_drive[Optical disc drive] - Digital camera - Digital music player They may be connected via any one of the following. - http://en.wikipedia.org/wiki/Universal_Serial_Bus[USB] - http://en.wikipedia.org/wiki/IEEE_1394[IEEE 1394 / FireWire] - http://en.wikipedia.org/wiki/PC_card[PC Card] Modern desktop environments such as GNOME and KDE can mount these removable devices automatically without a matching "`/etc/fstab`" entry. - `udisks` package provides a daemon and associated utilities to mount and unmount these devices. - http://en.wikipedia.org/wiki/D-Bus[D-bus] creates events to initiate automatic processes. - http://en.wikipedia.org/wiki/PolicyKit[PolicyKit] provides required privileges. TIP: Automounted devices may have the "`uhelper=`" mount option which is used by `umount`(8). TIP: Automounting under modern desktop environment happens only when those removable media devices are not listed in "`/etc/fstab`". Mount point under modern desktop environment is chosen as "`/media/`" which can be customized by the following. - `mlabel`(1) for FAT filesystem - `genisoimage`(1) with "`-V`" option for ISO9660 filesystem - `tune2fs`(1) with "`-L`" option for ext2/ext3/ext4 filesystem TIP: The choice of encoding may need to be provided as mount option (see <<_filename_encoding>>). ==== Filesystem choice for sharing data When sharing data with other system via removable storage device, you should format it with common http://en.wikipedia.org/wiki/File_system[filesystem] supported by both systems. Here is a list of filesystem choices. .List of filesystem choices for removable storage devices with typical usage scenarios [grid="all"] `----------------------------------------------------------------`------------------------------------------------------------------------ filesystem description of typical usage scenario ------------------------------------------------------------------------------------------------------------------------------------------ http://en.wikipedia.org/wiki/File_Allocation_Table[FAT12] cross platform sharing of data on the floppy disk (<32MiB) http://en.wikipedia.org/wiki/File_Allocation_Table[FAT16] cross platform sharing of data on the small hard disk like device (<2GiB) http://en.wikipedia.org/wiki/File_Allocation_Table[FAT32] cross platform sharing of data on the large hard disk like device (<8TiB, supported by newer than MS Windows95 OSR2) http://en.wikipedia.org/wiki/NTFS[NTFS] cross platform sharing of data on the large hard disk like device (supported natively on http://en.wikipedia.org/wiki/Windows_NT[MS Windows NT] and later version, and supported by http://en.wikipedia.org/wiki/NTFS-3G[NTFS-3G] via http://en.wikipedia.org/wiki/Filesystem_in_Userspace[FUSE] on Linux) http://en.wikipedia.org/wiki/ISO_9660[ISO9660] cross platform sharing of static data on CD-R and DVD+/-R http://en.wikipedia.org/wiki/Universal_Disk_Format[UDF] incremental data writing on CD-R and DVD+/-R (new) http://en.wikipedia.org/wiki/Minix_file_system[MINIX filesystem] space efficient unix file data storage on the floppy disk http://en.wikipedia.org/wiki/Ext2[ext2 filesystem] sharing of data on the hard disk like device with older Linux systems http://en.wikipedia.org/wiki/Ext3[ext3 filesystem] sharing of data on the hard disk like device with older Linux systems http://en.wikipedia.org/wiki/Ext4[ext4 filesystem] sharing of data on the hard disk like device with current Linux systems ------------------------------------------------------------------------------------------------------------------------------------------ TIP: See <<_removable_disk_encryption_with_dm_crypt_luks>> for cross platform sharing of data using device level encryption. The FAT filesystem is supported by almost all modern operating systems and is quite useful for the data exchange purpose via removable hard disk like media. When formatting removable hard disk like devices for cross platform sharing of data with the FAT filesystem, the following should be safe choices. - Partitioning them with `fdisk`(8), `cfdisk`(8) or `parted`(8) (see <<_disk_partition_configuration>>) into a single primary partition and to mark it as the following. * Type "6" for FAT16 for media smaller than 2GB. * Type "c" for FAT32 (LBA) for larger media. - Formatting the primary partition with `mkfs.vfat`(8) with the following. * Just its device name, e.g. "`/dev/sda1`" for FAT16 * The explicit option and its device name, e.g. "`-F 32 /dev/sda1`" for FAT32 When using the FAT or ISO9660 filesystems for sharing data, the following should be the safe considerations. - Archiving files into an archive file first using `tar`(1), or `cpio`(1) to retain the long filename, the symbolic link, the original Unix file permission and the owner information. - Splitting the archive file into less than 2 GiB chunks with the `split`(1) command to protect it from the file size limitation. - Encrypting the archive file to secure its contents from the unauthorized access. NOTE: For FAT filesystems by its design, the maximum file size is `(2\^32 - 1) bytes = (4GiB - 1 byte)`. For some applications on the older 32 bit OS, the maximum file size was even smaller `(2\^31 - 1) bytes = (2GiB - 1 byte)`. Debian does not suffer the latter problem. NOTE: Microsoft itself does not recommend to use FAT for drives or partitions of over 200 MB. Microsoft highlights its short comings such as inefficient disk space usage in their "http://support.microsoft.com/kb/100108/[Overview of FAT, HPFS, and NTFS File Systems]". Of course, we should normally use the ext4 filesystem for Linux. TIP: For more on filesystems and accessing filesystems, please read "http://tldp.org/HOWTO/Filesystems-HOWTO.html[Filesystems HOWTO]". ==== Sharing data via network When sharing data with other system via network, you should use common service. Here are some hints. .List of the network service to chose with the typical usage scenario [grid="all"] `------------------------------------------------------------------------------------------------------------------------------------------------`----------------------------------------------------------------------------------------------------------------------------------- network service description of typical usage scenario ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- http://en.wikipedia.org/wiki/Server_Message_Block[SMB/CIFS] network mounted filesystem with http://en.wikipedia.org/wiki/Samba_(software)[Samba] sharing files via "Microsoft Windows Network", see `smb.conf`(5) and http://www.samba.org/samba/docs/man/Samba-HOWTO-Collection/[The Official Samba 3.2.x HOWTO and Reference Guide] or the `samba-doc` package http://en.wikipedia.org/wiki/Network_File_System_(protocol)[NFS] network mounted filesystem with the Linux kernel sharing files via "Unix/Linux Network", see `exports`(5) and http://tldp.org/HOWTO/NFS-HOWTO/index.html[Linux NFS-HOWTO] http://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol[HTTP] service sharing file between the web server/client http://en.wikipedia.org/wiki/Https[HTTPS] service sharing file between the web server/client with encrypted Secure Sockets Layer (SSL) or http://en.wikipedia.org/wiki/Transport_Layer_Security[Transport Layer Security] (TLS) http://en.wikipedia.org/wiki/File_Transfer_Protocol[FTP] service sharing file between the FTP server/client ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Although these filesystems mounted over network and file transfer methods over network are quite convenient for sharing data, these may be insecure. Their network connection must be secured by the following. - Encrypt it with http://en.wikipedia.org/wiki/Transport_Layer_Security[SSL/TLS] - Tunnel it via http://en.wikipedia.org/wiki/Secure_Shell[SSH] - Tunnel it via http://en.wikipedia.org/wiki/Virtual_private_network[VPN] - Limit it behind the secure firewall See also <<_other_network_application_servers>> and <<_other_network_application_clients>>. === Backup and recovery We all know that computers fail sometime or human errors cause system and data damages. Backup and recovery operations are the essential part of successful system administration. All possible failure modes hit you some day. TIP: Keep your backup system simple and backup your system often. Having backup data is more important than how technically good your backup method is. There are 3 key factors which determine actual backup and recovery policy. 1. Knowing what to backup and recover. * Data files directly created by you: data in "`\~/`" * Data files created by applications used by you: data in "`/var/`" (except "`/var/cache/`", "`/var/run/`", and "`/var/tmp/`") * System configuration files: data in "`/etc/`" * Local softwares: data in "`/usr/local/`" or "`/opt/`" * System installation information: a memo in plain text on key steps (partition, ...) * Proven set of data: confirmed by experimental recovery operations in advance 2. Knowing how to backup and recover. * Secure storage of data: protection from overwrite and system failure * Frequent backup: scheduled backup * Redundant backup: data mirroring * Fool proof process: easy single command backup 3. Assessing risks and costs involved. * Value of data when lost * Required resources for backup: human, hardware, software, ... * Failure mode and their possibility NOTE: Do not back up the pseudo-filesystem contents found on `/proc`, `/sys`, `/tmp`, and `/run` (see <<_procfs_and_sysfs>> and <<_tmpfs>>). Unless you know exactly what you are doing, they are huge useless data. As for secure storage of data, data should be at least on different disk partitions preferably on different disks and machines to withstand the filesystem corruption. Important data are best stored on a write-once media such as CD/DVD-R to prevent overwrite accidents. (See <<_the_binary_data>> for how to write to the storage media from the shell commandline. GNOME desktop GUI environment gives you easy access via menu: "Places->CD/DVD Creator".) NOTE: You may wish to stop some application daemons such as MTA (see <<_mail_transport_agent_mta>>) while backing up data. NOTE: You should pay extra care to the backup and restoration of identity related data files such as "`/etc/ssh/ssh_host_dsa_key`", "`/etc/ssh/ssh_host_rsa_key`", "`\~/.gnupg/\*`", "`\~/.ssh/\*`", "`/etc/passwd`", "`/etc/shadow`", "`/etc/fetchmailrc`", "`popularity-contest.conf`", "`/etc/ppp/pap-secrets`", and "`/etc/exim4/passwd.client`". Some of these data can not be regenerated by entering the same input string to the system. NOTE: If you run a cron job as a user process, you must restore files in "`/var/spool/cron/crontabs`" directory and restart `cron`(8). See <<_scheduling_tasks_regularly>> for `cron`(8) and `crontab`(1). ==== Backup utility suites Here is a select list of notable backup utility suites available on the Debian system. .List of backup suite utilities [grid="all"] `----------------`-------------`------------`--------------------------------------------------------------------------------------------- package popcon size description ------------------------------------------------------------------------------------------------------------------------------------------ `dump` @-@popcon1@-@ @-@psize1@-@ 4.4 http://en.wikipedia.org/wiki/Berkeley_Software_Distribution[BSD] `dump`(8) and `restore`(8) for http://en.wikipedia.org/wiki/Ext2[ext2]/http://en.wikipedia.org/wiki/Ext3[ext3]/http://en.wikipedia.org/wiki/Ext4[ext4] filesystems `xfsdump` @-@popcon1@-@ @-@psize1@-@ dump and restore with `xfsdump`(8) and `xfsrestore`(8) for http://en.wikipedia.org/wiki/XFS[XFS] filesystem on GNU/Linux and http://en.wikipedia.org/wiki/IRIX[IRIX] `backupninja` @-@popcon1@-@ @-@psize1@-@ lightweight, extensible **meta-backup** system `sbackup` @-@popcon1@-@ @-@psize1@-@ simple backup suite for GNOME desktop `bacula-common` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Bacula[Bacula]: network backup, recovery and verification - common support files `bacula-client` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Bacula[Bacula]: network backup, recovery and verification - client meta-package `bacula-console` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Bacula[Bacula]: network backup, recovery and verification - text console `bacula-server` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Bacula[Bacula]: network backup, recovery and verification - server meta-package `amanda-common` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Advanced_Maryland_Automatic_Network_Disk_Archiver[Amanda]: Advanced Maryland Automatic Network Disk Archiver (Libs) `amanda-client` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Advanced_Maryland_Automatic_Network_Disk_Archiver[Amanda]: Advanced Maryland Automatic Network Disk Archiver (Client) `amanda-server` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Advanced_Maryland_Automatic_Network_Disk_Archiver[Amanda]: Advanced Maryland Automatic Network Disk Archiver (Server) `backuppc` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Backuppc[BackupPC] is a high-performance, enterprise-grade system for backing up PCs (disk based) `backup-manager` @-@popcon1@-@ @-@psize1@-@ command-line backup tool `backup2l` @-@popcon1@-@ @-@psize1@-@ low-maintenance backup/restore tool for mountable media (disk based) ------------------------------------------------------------------------------------------------------------------------------------------ Backup tools have their specialized focuses. - http://en.wikipedia.org/wiki/Mondo_Rescue[Mondo Rescue] is a backup system to facilitate restoration of complete system quickly from backup CD/DVD etc. without going through normal system installation processes. - `sbackup` and `keep` packages provide easy GUI frontend for desktop users to make regular backups of user data. An equivalent function can be realized by a simple script (<<_an_example_script_for_the_system_backup>>) and `cron`(8). - http://en.wikipedia.org/wiki/Bacula[Bacula], http://en.wikipedia.org/wiki/Advanced_Maryland_Automatic_Network_Disk_Archiver[Amanda], and http://en.wikipedia.org/wiki/Backuppc[BackupPC] are full featured backup suite utilities which are focused on regular backups over network. Basic tools described in <<_archive_and_compression_tools>> and <<_copy_and_synchronization_tools>> can be used to facilitate system backup via custom scripts. Such script can be enhanced by the following. - The `obnam` package enables incremental (remote) backups. - The `rdiff-backup` package enables incremental (remote) backups. - The `dump` package helps to archive and restore the whole filesystem incrementally and efficiently. TIP: See files in "`/usr/share/doc/dump/`" and http://dump.sourceforge.net/isdumpdeprecated.html["Is dump really deprecated?"] to lean about the `dump` package. ==== An example script for the system backup For a personal Debian desktop system running `unstable` suite, I only need to protect personal and critical data. I reinstall system once a year anyway. Thus I see no reason to backup the whole system or to install a full featured backup utility. I use a simple script to make a backup archive and burn it into CD/DVD using GUI. Here is an example script for this. -------------------- #!/bin/sh -e # Copyright (C) 2007-2008 Osamu Aoki , Public Domain BUUID=1000; USER=osamu # UID and name of a user who accesses backup files BUDIR="/var/backups" XDIR0=".+/Mail|.+/Desktop" XDIR1=".+/\.thumbnails|.+/\.?Trash|.+/\.?[cC]ache|.+/\.gvfs|.+/sessions" XDIR2=".+/CVS|.+/\.git|.+/\.svn|.+/Downloads|.+/Archive|.+/Checkout|.+/tmp" XSFX=".+\.iso|.+\.tgz|.+\.tar\.gz|.+\.tar\.bz2|.+\.cpio|.+\.tmp|.+\.swp|.+~" SIZE="+99M" DATE=$(date --utc +"%Y%m%d-%H%M") [ -d "$BUDIR" ] || mkdir -p "BUDIR" umask 077 dpkg --get-selections \* > /var/lib/dpkg/dpkg-selections.list debconf-get-selections > /var/cache/debconf/debconf-selections { find /etc /usr/local /opt /var/lib/dpkg/dpkg-selections.list \ /var/cache/debconf/debconf-selections -xdev -print0 find /home/$USER /root -xdev -regextype posix-extended \ -type d -regex "$XDIR0|$XDIR1" -prune -o -type f -regex "$XSFX" -prune -o \ -type f -size "$SIZE" -prune -o -print0 find /home/$USER/Mail/Inbox /home/$USER/Mail/Outbox -print0 find /home/$USER/Desktop -xdev -regextype posix-extended \ -type d -regex "$XDIR2" -prune -o -type f -regex "$XSFX" -prune -o \ -type f -size "$SIZE" -prune -o -print0 } | cpio -ov --null -O $BUDIR/BU$DATE.cpio chown $BUUID $BUDIR/BU$DATE.cpio touch $BUDIR/backup.stamp -------------------- This is meant to be a script example executed from root. I expect you to change and execute this as follows. - Edit this script to cover all your important data (see <<_idioms_for_the_selection_of_files>> and <<_backup_and_recovery>>). - Replace "`find ... -print0`" with "`find ... -newer $BUDIR/backup.stamp -print0`" to make a incremental backup. - Transfer backup files to the remote host using `scp`(1) or `rsync`(1) or burn them to CD/DVD for extra data security. (I use GNOME desktop GUI for burning CD/DVD. See <<_shell_script_example_with_zenity>> for extra redundancy.) Keep it simple! TIP: You can recover debconf configuration data with "`debconf-set-selections debconf-selections`" and dpkg selection data with "`dpkg --set-selection , Public Domain fdot(){ find . -type d \( -iname ".?*" -o -iname "CVS" \) -prune -o -print0;} fall(){ find . -print0;} mkdircd(){ mkdir -p "$1";chmod 700 "$1";cd "$1">/dev/null;} FIND="fdot";OPT="-a";MODE="CPIOP";HOST="localhost";EXTP="$(hostname -f)" BKUP="$(basename $(pwd)).bkup";TIME="$(date +%Y%m%d-%H%M%S)";BU="$BKUP/$TIME" while getopts gcCsStrlLaAxe:h:T f; do case $f in g) MODE="GNUCP";; # cp (GNU) c) MODE="CPIOP";; # cpio -p C) MODE="CPIOI";; # cpio -i s) MODE="CPIOSSH";; # cpio/ssh t) MODE="TARSSH";; # tar/ssh r) MODE="RSYNCSSH";; # rsync/ssh l) OPT="-alv";; # hardlink (GNU cp) L) OPT="-av";; # copy (GNU cp) a) FIND="fall";; # find all A) FIND="fdot";; # find non CVS/ .???/ x) set -x;; # trace e) EXTP="${OPTARG}";; # hostname -f h) HOST="${OPTARG}";; # user@remotehost.example.com T) MODE="TEST";; # test find mode \?) echo "use -x for trace." esac; done shift $(expr $OPTIND - 1) if [ $# -gt 0 ]; then for x in $@; do cp $OPT $x $x.$TIME; done elif [ $MODE = GNUCP ]; then mkdir -p "../$BU";chmod 700 "../$BU";cp $OPT . "../$BU/" elif [ $MODE = CPIOP ]; then mkdir -p "../$BU";chmod 700 "../$BU" $FIND|cpio --null --sparse -pvd ../$BU elif [ $MODE = CPIOI ]; then $FIND|cpio -ov --null | ( mkdircd "../$BU"&&cpio -i ) elif [ $MODE = CPIOSSH ]; then $FIND|cpio -ov --null|ssh -C $HOST "( mkdircd \"$EXTP/$BU\"&&cpio -i )" elif [ $MODE = TARSSH ]; then (tar cvf - . )|ssh -C $HOST "( mkdircd \"$EXTP/$BU\"&& tar xvfp - )" elif [ $MODE = RSYNCSSH ]; then rsync -aHAXSv ./ "${HOST}:${EXTP}-${BKUP}-${TIME}" else echo "Any other idea to backup?" $FIND |xargs -0 -n 1 echo fi -------------------- This is meant to be command examples. Please read script and edit it by yourself before using it. TIP: I keep this `bkup` in my "`/usr/local/bin/`" directory. I issue this `bkup` command without any option in the working directory whenever I need a temporary snapshot backup. TIP: For making snapshot history of a source file tree or a configuration file tree, it is easier and space efficient to use `git`(7) (see <<_git_for_recording_configuration_history>>). === Data security infrastructure The data security infrastructure is provided by the combination of data encryption tool, message digest tool, and signature tool. .List of data security infrastructure tools [grid="all"] `----------------`-------------`------------`--------------------`------------------------------------------------------------------------ package popcon size command description ------------------------------------------------------------------------------------------------------------------------------------------ `gnupg` @-@popcon1@-@ @-@psize1@-@ `gpg`(1) http://en.wikipedia.org/wiki/GNU_Privacy_Guard[GNU Privacy Guard] - OpenPGP encryption and signing tool `gnupg-doc` @-@popcon1@-@ @-@psize1@-@ N/A GNU Privacy Guard documentation `gpgv` @-@popcon1@-@ @-@psize1@-@ `gpgv`(1) GNU Privacy Guard - signature verification tool `paperkey` @-@popcon1@-@ @-@psize1@-@ `paperkey`(1) extract just the secret information out of OpenPGP secret keys `cryptsetup` @-@popcon1@-@ @-@psize1@-@ `cryptsetup`(8), ... utilities for http://en.wikipedia.org/wiki/Dm-crypt[dm-crypto] block device encryption supporting http://en.wikipedia.org/wiki/Linux_Unified_Key_Setup[LUKS] `ecryptfs-utils` @-@popcon1@-@ @-@psize1@-@ `ecryptfs`(7), ... utilities for http://ecryptfs.sourceforge.net/[ecryptfs] stacked filesystem encryption `coreutils` @-@popcon1@-@ @-@psize1@-@ `md5sum`(1) compute and check MD5 message digest `coreutils` @-@popcon1@-@ @-@psize1@-@ `sha1sum`(1) compute and checks SHA1 message digest `openssl` @-@popcon1@-@ @-@psize1@-@ `openssl`(1ssl) compute message digest with "`openssl dgst`" (OpenSSL) ------------------------------------------------------------------------------------------------------------------------------------------ See <<_data_encryption_tips>> on http://en.wikipedia.org/wiki/Dm-crypt[dm-crypto] and http://ecryptfs.sourceforge.net/[ecryptfs] which implement automatic data encryption infrastructure via Linux kernel modules. ==== Key management for GnuPG Here are http://en.wikipedia.org/wiki/GNU_Privacy_Guard[GNU Privacy Guard] commands for the basic key management. .List of GNU Privacy Guard commands for the key management [grid="all"] `-----------------------------`--------------------------------------- command description ---------------------------------------------------------------------- `gpg --gen-key` generate a new key `gpg --gen-revoke my_user_ID` generate revoke key for my_user_ID `gpg --edit-key user_ID` edit key interactively, "help" for help `gpg -o file --exports` export all keys to file `gpg --imports file` import all keys from file `gpg --send-keys user_ID` send key of user_ID to keyserver `gpg --recv-keys user_ID` recv. key of user_ID from keyserver `gpg --list-keys user_ID` list keys of user_ID `gpg --list-sigs user_ID` list sig. of user_ID `gpg --check-sigs user_ID` check sig. of user_ID `gpg --fingerprint user_ID` check fingerprint of user_ID `gpg --refresh-keys` update local keyring ---------------------------------------------------------------------- Here is the meaning of the trust code. .List of the meaning of the trust code [grid="all"] `----`--------------------------------------------- code description of trust --------------------------------------------------- `-` no owner trust assigned / not yet calculated `e` trust calculation failed `q` not enough information for calculation `n` never trust this key `m` marginally trusted `f` fully trusted `u` ultimately trusted --------------------------------------------------- The following uploads my key "`1DD8D791`" to the popular keyserver "`hkp://keys.gnupg.net`". -------------------- $ gpg --keyserver hkp://keys.gnupg.net --send-keys 1DD8D791 -------------------- A good default keyserver set up in "`\~/.gnupg/gpg.conf`" (or old location "`\~/.gnupg/options`") contains the following. -------------------- keyserver hkp://keys.gnupg.net -------------------- The following obtains unknown keys from the keyserver. -------------------- $ gpg --list-sigs --with-colons | grep '^sig.*\[User ID not found\]' |\ cut -d ':' -f 5| sort | uniq | xargs gpg --recv-keys -------------------- There was a bug in http://sourceforge.net/projects/pks/[OpenPGP Public Key Server] (pre version 0.9.6) which corrupted key with more than 2 sub-keys. The newer `gnupg` (>1.2.1-2) package can handle these corrupted subkeys. See `gpg`(1) under "`--repair-pks-subkey-bug`" option. ==== Using GnuPG on files Here are examples for using http://en.wikipedia.org/wiki/GNU_Privacy_Guard[GNU Privacy Guard] commands on files. .List of GNU Privacy Guard commands on files [grid="all"] `-------------------------------------------------------`--------------------------------------------------------------------------------- command description ------------------------------------------------------------------------------------------------------------------------------------------ `gpg -a -s file` sign file into http://en.wikipedia.org/wiki/ASCII[ASCII] armored file.asc `gpg --armor --sign file` , , `gpg --clearsign file` clear-sign message `gpg --clearsign file|mail foo@example.org` mail a clear-signed message to `foo@example.org` `gpg --clearsign --not-dash-escaped patchfile` clear-sign patchfile `gpg --verify file` verify clear-signed file `gpg -o file.sig -b file` create detached signature `gpg -o file.sig --detach-sig file` , , `gpg --verify file.sig file` verify file with file.sig `gpg -o crypt_file.gpg -r name -e file` public-key encryption intended for name from file to binary crypt_file.gpg `gpg -o crypt_file.gpg --recipient name --encrypt file` , , `gpg -o crypt_file.asc -a -r name -e file` public-key encryption intended for name from file to http://en.wikipedia.org/wiki/ASCII[ASCII] armored crypt_file.asc `gpg -o crypt_file.gpg -c file` symmetric encryption from file to crypt_file.gpg `gpg -o crypt_file.gpg --symmetric file` , , `gpg -o crypt_file.asc -a -c file` symmetric encryption intended for name from file to http://en.wikipedia.org/wiki/ASCII[ASCII] armored crypt_file.asc `gpg -o file -d crypt_file.gpg -r name` decryption `gpg -o file --decrypt crypt_file.gpg` , , ------------------------------------------------------------------------------------------------------------------------------------------ ==== Using GnuPG with Mutt Add the following to "`\~/.muttrc`" to keep a slow GnuPG from automatically starting, while allowing it to be used by typing "`S`" at the index menu. -------------------- macro index S ":toggle pgp_verify_sig\n" set pgp_verify_sig=no -------------------- ==== Using GnuPG with Vim The `gnupg` plugin let you run GnuPG transparently for files with extension "`.gpg`", "`.asc`", and "`.ppg`". -------------------- # aptitude install vim-scripts vim-addon-manager $ vim-addons install gnupg -------------------- ==== The MD5 sum `md5sum`(1) provides utility to make a digest file using the method in http://tools.ietf.org/html/rfc1321[rfc1321] and verifying each file with it. -------------------- $ md5sum foo bar >baz.md5 $ cat baz.md5 d3b07384d113edec49eaa6238ad5ff00 foo c157a79031e1c40f85931829bc5fc552 bar $ md5sum -c baz.md5 foo: OK bar: OK -------------------- NOTE: The computation for the http://en.wikipedia.org/wiki/MD5[MD5] sum is less CPU intensive than the one for the cryptographic signature by http://en.wikipedia.org/wiki/GNU_Privacy_Guard[GNU Privacy Guard (GnuPG)]. Usually, only the top level digest file is cryptographically signed to ensure data integrity. === Source code merge tools There are many merge tools for the source code. Following commands caught my eyes. .List of source code merge tools [grid="all"] `------------`-------------`------------`----------------`----------------------------------------------------------------- package popcon size command description --------------------------------------------------------------------------------------------------------------------------- `diffutils` @-@popcon1@-@ @-@psize1@-@ `diff`(1) compare files line by line `diffutils` @-@popcon1@-@ @-@psize1@-@ `diff3`(1) compare and merges three files line by line `vim` @-@popcon1@-@ @-@psize1@-@ `vimdiff`(1) compare 2 files side by side in vim `patch` @-@popcon1@-@ @-@psize1@-@ `patch`(1) apply a diff file to an original `dpatch` @-@popcon1@-@ @-@psize1@-@ `dpatch`(1) manage series of patches for Debian package `diffstat` @-@popcon1@-@ @-@psize1@-@ `diffstat`(1) produce a histogram of changes by the diff `patchutils` @-@popcon1@-@ @-@psize1@-@ `combinediff`(1) create a cumulative patch from two incremental patches `patchutils` @-@popcon1@-@ @-@psize1@-@ `dehtmldiff`(1) extract a diff from an HTML page `patchutils` @-@popcon1@-@ @-@psize1@-@ `filterdiff`(1) extract or excludes diffs from a diff file `patchutils` @-@popcon1@-@ @-@psize1@-@ `fixcvsdiff`(1) fix diff files created by CVS that `patch`(1) mis-interprets `patchutils` @-@popcon1@-@ @-@psize1@-@ `flipdiff`(1) exchange the order of two patches `patchutils` @-@popcon1@-@ @-@psize1@-@ `grepdiff`(1) show which files are modified by a patch matching a regex `patchutils` @-@popcon1@-@ @-@psize1@-@ `interdiff`(1) show differences between two unified diff files `patchutils` @-@popcon1@-@ @-@psize1@-@ `lsdiff`(1) show which files are modified by a patch `patchutils` @-@popcon1@-@ @-@psize1@-@ `recountdiff`(1) recompute counts and offsets in unified context diffs `patchutils` @-@popcon1@-@ @-@psize1@-@ `rediff`(1) fix offsets and counts of a hand-edited diff `patchutils` @-@popcon1@-@ @-@psize1@-@ `splitdiff`(1) separate out incremental patches `patchutils` @-@popcon1@-@ @-@psize1@-@ `unwrapdiff`(1) demangle patches that have been word-wrapped `wiggle` @-@popcon1@-@ @-@psize1@-@ `wiggle`(1) apply rejected patches `quilt` @-@popcon1@-@ @-@psize1@-@ `quilt`(1) manage series of patches `meld` @-@popcon1@-@ @-@psize1@-@ `meld`(1) compare and merge files (GTK) `dirdiff` @-@popcon1@-@ @-@psize1@-@ `dirdiff`(1) display differences and merge changes between directory trees `docdiff` @-@popcon1@-@ @-@psize1@-@ `docdiff`(1) compare two files word by word / char by char `imediff2` @-@popcon1@-@ @-@psize1@-@ `imediff2`(1) interactive full screen 2-way merge tool `makepatch` @-@popcon1@-@ @-@psize1@-@ `makepatch`(1) generate extended patch files `makepatch` @-@popcon1@-@ @-@psize1@-@ `applypatch`(1) apply extended patch files `wdiff` @-@popcon1@-@ @-@psize1@-@ `wdiff`(1) display word differences between text files --------------------------------------------------------------------------------------------------------------------------- ==== Extracting differences for source files One of following procedures extract differences between two source files and create unified diff files "`file.patch0`" or "`file.patch1`" depending on the file location. -------------------- $ diff -u file.old file.new > file.patch0 $ diff -u old/file new/file > file.patch1 -------------------- ==== Merging updates for source files The diff file (alternatively called patch file) is used to send a program update. The receiving party applies this update to another file by the following. -------------------- $ patch -p0 file < file.patch0 $ patch -p1 file < file.patch1 -------------------- ==== Updating via 3-way-merge If you have three versions of a source code, you can perform 3-way-merge effectively using `diff3`(1) by the following. -------------------- $ diff3 -m file.mine file.old file.yours > file -------------------- === Version control systems Here is a summary of the http://en.wikipedia.org/wiki/Revision_control[version control systems (VCS)] on the Debian system. NOTE: If you are new to VCS systems, you should start learning with **Git**, which is growing fast in popularity. [[list-of-vcs]] .List of version control system tools [grid="all"] `------------`-------------`------------`--------------------------------------------------------------`-----------`-------------------------------------------------------------------------------------------- package popcon size tool VCS type comment ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- `cssc` @-@popcon1@-@ @-@psize1@-@ http://cssc.sourceforge.net/[CSSC] local clone of the http://en.wikipedia.org/wiki/Source_Code_Control_System[Unix SCCS] (deprecated) `rcs` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Revision_Control_System[RCS] local "http://en.wikipedia.org/wiki/Source_Code_Control_System[Unix SCCS] done right" `cvs` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Concurrent_Versions_System[CVS] remote previous standard remote VCS `subversion` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Subversion_(software)[Subversion] remote "CVS done right", the new de facto standard remote VCS `git` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Git_(software)[Git] distributed fast DVCS in C (used by the Linux kernel and others) `mercurial` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Mercurial_(software)[Mercurial] distributed DVCS in Python and some C `bzr` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Bazaar_(software)[Bazaar] distributed DVCS influenced by `tla` written in Python (used by http://www.ubuntu.com/[Ubuntu]) `darcs` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Darcs[Darcs] distributed DVCS with smart algebra of patches (slow) `tla` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/GNU_arch[GNU arch] distributed DVCS mainly by Tom Lord (Historic) `monotone` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/Monotone_(software)[Monotone] distributed DVCS in C++ `tkcvs` @-@popcon1@-@ @-@psize1@-@ CVS, ... remote GUI display of VCS (CVS, Subversion, RCS) repository tree `gitk` @-@popcon1@-@ @-@psize1@-@ Git distributed GUI display of VCS (Git) repository tree ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- VCS is sometimes known as revision control system (RCS), or software configuration management (SCM). Distributed VCS such as Git is the tool of choice these days. CVS and Subversion may still be useful to join some existing open source program activities. Debian provides free VCS services via http://alioth.debian.org/[Debian Alioth service]. It supports practically all VCSs. Its documentation can be found at http://wiki.debian.org/Alioth . There are few basics for creating a shared access VCS archive. - Use "`umask 002`" (see <<_control_of_permissions_for_newly_created_files_umask>>) - Make all VCS archive files belonging to a pertinent group - Enable set group ID on all VCS archive directories (BSD-like file creation scheme, see <<_filesystem_permissions>>) - Make user sharing the VCS archive belonging to the group ==== Comparison of VCS commands Here is an oversimplified comparison of native VCS commands to provide the big picture. The typical command sequence may require options and arguments. .Comparison of native VCS commands [grid="all"] `-----------------------------`------------`------------`-------------------------------------------------------- Git CVS Subversion function ----------------------------------------------------------------------------------------------------------------- `git init` `cvs init` `svn create` create the (local) repository - `cvs login` - login to the remote repository `git clone` `cvs co` `svn co` check out the remote repository as the working tree `git pull` `cvs up` `svn up` update the working tree by merging the remote repository `git add .` `cvs add` `svn add` add file(s) in the working tree to the VCS `git rm` `cvs rm` `svn rm` remove file(s) in working tree from the VCS - `cvs ci` `svn ci` commit changes to the remote repository `git commit -a` - - commit changes to the local repository `git push` - - update the remote repository by the local repository `git status` `cvs status` `svn status` display the working tree status from the VCS `git diff` `cvs diff` `svn diff` diff `git repack -a -d; git prune` - - repack the local repository into single pack `gitk` `tkcvs` `tkcvs` GUI display of VCS repository tree ----------------------------------------------------------------------------------------------------------------- CAUTION: Invoking a `git` subcommand directly as "`git-xyz`" from the command line has been deprecated since early 2006. TIP: GUI tools such as `tkcvs`(1) and `gitk`(1) really help you with tracking revision history of files. The web interface provided by many public archives for browsing their repositories is also quite useful, too. TIP: Git can work directly with different VCS repositories such as ones provided by CVS and Subversion, and provides the local repository for local changes with `git-cvs` and `git-svn` packages. See http://www.kernel.org/pub/software/scm/git/docs/gitcvs-migration.html[git for CVS users], and <<_git_for_the_subversion_repository>>. TIP: Git has commands which have no equivalents in CVS and Subversion: "fetch", "rebase", "cherry-pick", ... // Following URLs are interesting. // [http://www.mantisbt.org/wiki/doku.php/mantisbt:git_svn Using Git with SVN] // [http://andy.delcambre.com/2008/3/4/git-svn-workflow Git SVN Workflow] // [http://www.gnome.org/~federico/misc/git-cheat-sheet.txt GIT for mortals] // [http://kerneltrap.org/mailarchive/git/2007/6/26/250068 GIT + CVS workflow query] // http://lwn.net/Articles/210045/ === Git Git can do everything for both local and remote source code management. This means that you can record the source code changes without needing network connectivity to the remote repository. ==== Configuration of Git client You may wish to set several global configuration in "`\~/.gitconfig`" such as your name and email address used by Git by the following. -------------------- $ git config --global user.name "Name Surname" $ git config --global user.email yourname@example.com -------------------- If you are too used to CVS or Subversion commands, you may wish to set several command aliases by the following. -------------------- $ git config --global alias.ci "commit -a" $ git config --global alias.co checkout -------------------- You can check your global configuration by the following. -------------------- $ git config --global --list -------------------- ==== Git references See the following. - http://www.kernel.org/pub/software/scm/git/docs/git.html[manpage: git(1)] (`/usr/share/doc/git-doc/git.html`) - http://www.kernel.org/pub/software/scm/git/docs/user-manual.html[Git User@@@sq@@@s Manual] (`/usr/share/doc/git-doc/user-manual.html`) - http://www.kernel.org/pub/software/scm/git/docs/gittutorial.html[A tutorial introduction to git] (`/usr/share/doc/git-doc/gittutorial.html`) - http://www.kernel.org/pub/software/scm/git/docs/gittutorial-2.html[A tutorial introduction to git: part two] (`/usr/share/doc/git-doc/gittutorial-2.html`) - http://www.kernel.org/pub/software/scm/git/docs/everyday.html[Everyday GIT With 20 Commands Or So] (`/usr/share/doc/git-doc/everyday.html`) - http://www.kernel.org/pub/software/scm/git/docs/gitcvs-migration.html[git for CVS users] (`/usr/share/doc/git-doc/gitcvs-migration.html`) * This also describes how to set up server like CVS and extract old data from CVS into Git. - http://git-scm.com/documentation[Other git resources available on the web] * http://git-scm.com/course/svn.html[Git - SVN Crash Course] * http://www-cs-students.stanford.edu/\~blynn/gitmagic/[Git Magic] (`/usr/share/doc/gitmagic/html/index.html`) `git-gui`(1) and `gitk`(1) commands make using Git very easy. WARNING: Do not use the tag string with spaces in it even if some tools such as `gitk`(1) allow you to use it. It may choke some other `git` commands. ==== Git commands Even if your upstream uses different VCS, it may be a good idea to use `git`(1) for local activity since you can manage your local copy of source tree without the network connection to the upstream. Here are some packages and commands used with `git`(1). .List of git related packages and commands [grid="all"] `------------------`-------------`------------`------------------------`------------------------------------------------------------------ package popcon size command description ------------------------------------------------------------------------------------------------------------------------------------------ `git-doc` @-@popcon1@-@ @-@psize1@-@ N/A official documentation for Git `gitmagic` @-@popcon1@-@ @-@psize1@-@ N/A "Git Magic", easier to understand guide for Git `git` @-@popcon1@-@ @-@psize1@-@ `git`(7) Git, the fast, scalable, distributed revision control system `gitk` @-@popcon1@-@ @-@psize1@-@ `gitk`(1) GUI Git repository browser with history `git-gui` @-@popcon1@-@ @-@psize1@-@ `git-gui`(1) GUI for Git (No history) `git-svn` @-@popcon1@-@ @-@psize1@-@ `git-svnimport`(1) import the data out of Subversion into Git `git-svn` @-@popcon1@-@ @-@psize1@-@ `git-svn`(1) provide bidirectional operation between the Subversion and Git `git-cvs` @-@popcon1@-@ @-@psize1@-@ `git-cvsimport`(1) import the data out of CVS into Git `git-cvs` @-@popcon1@-@ @-@psize1@-@ `git-cvsexportcommit`(1) export a commit to a CVS checkout from Git `git-cvs` @-@popcon1@-@ @-@psize1@-@ `git-cvsserver`(1) CVS server emulator for Git `git-email` @-@popcon1@-@ @-@psize1@-@ `git-send-email`(1) send a collection of patches as email from the Git `stgit` @-@popcon1@-@ @-@psize1@-@ `stg`(1) quilt on top of git (Python) `git-buildpackage` @-@popcon1@-@ @-@psize1@-@ `git-buildpackage`(1) automate the Debian packaging with the Git `guilt` @-@popcon1@-@ @-@psize1@-@ `guilt`(7) quilt on top of git (SH/AWK/SED/...) ------------------------------------------------------------------------------------------------------------------------------------------ TIP: With `git`(1), you work on a local branch with many commits and use something like "`git rebase -i master`" to reorganize change history later. This enables you to make clean change history. See `git-rebase`(1) and `git-cherry-pick`(1). TIP: When you want to go back to a clean working directory without loosing the current state of the working directory, you can use "`git stash`". See `git-stash`(1). ==== Git for the Subversion repository You can check out a Subversion repository at "`svn+ssh://svn.example.org/project/module/trunk`" to a local Git repository at "`./dest`" and commit back to the Subversion repository. E.g.: -------------------- $ git svn clone -s -rHEAD svn+ssh://svn.example.org/project dest $ cd dest ... make changes $ git commit -a ... keep working locally with git $ git svn dcommit -------------------- TIP: The use of "`-rHEAD`" enables us to avoid cloning entire historical contents from the Subversion repository. ==== Git for recording configuration history You can manually record chronological history of configuration using http://en.wikipedia.org/wiki/Git_(software)[Git] tools. Here is a simple example for your practice to record "`/etc/apt/`" contents. // sudo environment is assumed for realistic scenario. // Please do not complain... -------------------- $ cd /etc/apt/ $ sudo git init $ sudo chmod 700 .git $ sudo git add . $ sudo git commit -a -------------------- Commit configuration with description. Make modification to the configuration files. -------------------- $ cd /etc/apt/ $ sudo git commit -a -------------------- Commit configuration with description and continue your life. -------------------- $ cd /etc/apt/ $ sudo gitk --all -------------------- You have full configuration history with you. NOTE: `sudo`(8) is needed to work with any file permissions of configuration data. For user configuration data, you may skip `sudo`. NOTE: The "`chmod 700 .git`" command in the above example is needed to protect archive data from unauthorized read access. TIP: For more complete setup for recording configuration history, please look for the `etckeeper` package: <<_recording_changes_in_configuration_files>>. === CVS See the following. - `cvs`(1) - "`/usr/share/doc/cvs/html-cvsclient`" - "`/usr/share/doc/cvs/html-info`" - "`/usr/share/doc/cvsbook`" - "`info cvs`" ==== Configuration of CVS repository The following configuration allows commits to the CVS repository only by a member of the "`src`" group, and administration of CVS only by a member of the "`staff`" group, thus reducing the chance of shooting oneself. -------------------- # cd /var/lib; umask 002; mkdir cvs # export CVSROOT=/srv/cvs/project # cd $CVSROOT # chown root:src . # chmod 2775 . # cvs -d $CVSROOT init # cd CVSROOT # chown -R root:staff . # chmod 2775 . # touch val-tags # chmod 664 history val-tags # chown root:src history val-tags -------------------- TIP: You may restrict creation of new project by changing the owner of "`$CVSROOT`" directory to "`root:staff`" and its permission to "`3775`". ==== Local access to CVS The default CVS repository is pointed by "`$CVSROOT`". The following sets up "`$CVSROOT`" for the local access. -------------------- $ export CVSROOT=/srv/cvs/project -------------------- ==== Remote access to CVS with pserver Many public CVS servers provide read-only remote access to them with account name "`anonymous`" via pserver service. For example, Debian web site contents are maintained by http://alioth.debian.org/projects/webwml/[webwml project] via CVS at Debian alioth service. The following sets up "`$CVSROOT`" for the remote access to this CVS repository. -------------------- $ export CVSROOT=:pserver:anonymous@cvs.alioth.debian.org:/cvsroot/webwml $ cvs login -------------------- NOTE: Since pserver is prone to eavesdropping attack and insecure, write access is usually disable by server administrators. ==== Remote access to CVS with ssh The following sets up "`$CVS_RSH`" and "`$CVSROOT`" for the remote access to the CVS repository by http://alioth.debian.org/projects/webwml/[webwml project] with SSH. -------------------- $ export CVS_RSH=ssh $ export CVSROOT=:ext:account@cvs.alioth.debian.org:/cvs/webwml -------------------- You can also use public key authentication for SSH which eliminates the remote password prompt. ==== Importing a new source to CVS Create a new local source tree location at "`~/path/to/module1`" by the following. -------------------- $ mkdir -p ~/path/to/module1; cd ~/path/to/module1 -------------------- Populate a new local source tree under "`~/path/to/module1`" with files. Import it to CVS with the following parameters. - Module name: "`module1`" - Vendor tag: "`Main-branch`" (tag for the entire branch) - Release tag: "`Release-initial`" (tag for a specific release) -------------------- $ cd ~/path/to/module1 $ cvs import -m "Start module1" module1 Main-branch Release-initial $ rm -Rf . # optional -------------------- ==== File permissions in CVS repository CVS does not overwrite the current repository file but replaces it with another one. Thus, write permission to the repository directory is critical. For every new module for "`module1`" in repository at "`/srv/cvs/project`", run the following to ensure this condition if needed. -------------------- # cd /srv/cvs/project # chown -R root:src module1 # chmod -R ug+rwX module1 # chmod 2775 module1 -------------------- ==== Work flow of CVS Here is an example of typical work flow using CVS. Check all available modules from CVS project pointed by "`$CVSROOT`" by the following. -------------------- $ cvs rls CVSROOT module1 module2 ... -------------------- Checkout "`module1`" to its default directory "`./module1`" by the following. -------------------- $ cd ~/path/to $ cvs co module1 $ cd module1 -------------------- Make changes to the content as needed. Check changes by making "`diff -u [repository] [local]`" equivalent by the following. -------------------- $ cvs diff -u -------------------- You find that you broke some file "`file_to_undo`" severely but other files are fine. Overwrite "`file_to_undo`" file with the clean copy from CVS by the following. -------------------- $ cvs up -C file_to_undo -------------------- Save the updated local source tree to CVS by the following. -------------------- $ cvs ci -m "Describe change" -------------------- Create and add "`file_to_add`" file to CVS by the following. -------------------- $ vi file_to_add $ cvs add file_to_add $ cvs ci -m "Added file_to_add" -------------------- Merge the latest version from CVS by the following. -------------------- $ cvs up -d -------------------- Watch out for lines starting with "`C filename`" which indicates conflicting changes. Look for unmodified code in "`.#filename.version`". Search for "`<<<<<<<`" and "`>>>>>>>`" in files for conflicting changes. Edit files to fix conflicts as needed. Add a release tag "`Release-1`" by the following. -------------------- $ cvs ci -m "last commit for Release-1" $ cvs tag Release-1 -------------------- Edit further. Remove the release tag "`Release-1`" by the following. -------------------- $ cvs tag -d Release-1 -------------------- Check in changes to CVS by the following. -------------------- $ cvs ci -m "real last commit for Release-1" -------------------- Re-add the release tag "`Release-1`" to updated CVS HEAD of main by the following. -------------------- $ cvs tag Release-1 -------------------- Create a branch with a sticky branch tag "`Release-initial-bugfixes`" from the original version pointed by the tag "`Release-initial`" and check it out to "`~/path/to/old`" directory by the following. -------------------- $ cvs rtag -b -r Release-initial Release-initial-bugfixes module1 $ cd ~/path/to $ cvs co -r Release-initial-bugfixes -d old module1 $ cd old -------------------- TIP: Use "`-D 2005-12-20`" (http://en.wikipedia.org/wiki/ISO_8601[ISO 8601] date format) instead of "`-r Release-initial`" to specify particular date as the branch point. Work on this local source tree having the sticky tag "`Release-initial-bugfixes`" which is based on the original version. Work on this branch by yourself ... until someone else joins to this "`Release-initial-bugfixes`" branch. Sync with files modified by others on this branch while creating new directories as needed by the following. -------------------- $ cvs up -d -------------------- Edit files to fix conflicts as needed. Check in changes to CVS by the following. -------------------- $ cvs ci -m "checked into this branch" -------------------- Update the local tree by HEAD of main while removing sticky tag ("`-A`") and without keyword expansion ("`-kk`") by the following. -------------------- $ cvs up -d -kk -A -------------------- Update the local tree (content = HEAD of main) by merging from the "`Release-initial-bugfixes`" branch and without keyword expansion by the following. -------------------- $ cvs up -d -kk -j Release-initial-bugfixes -------------------- Fix conflicts with editor. Check in changes to CVS by the following. -------------------- $ cvs ci -m "merged Release-initial-bugfixes" -------------------- Make archive by the following. -------------------- $ cd .. $ mv old old-module1-bugfixes $ tar -cvzf old-module1-bugfixes.tar.gz old-module1-bugfixes $ rm -rf old-module1-bugfixes -------------------- TIP: "`cvs up`" command can take "`-d`" option to create new directories and "`-P`" option to prune empty directories. TIP: You can checkout only a sub directory of "`module1`" by providing its name as "`cvs co module1/subdir`". .Notable options for CVS commands (use as first argument(s) to `cvs`(1)) [grid="all"] `------`---------------------------------------------- option meaning ------------------------------------------------------ `-n` dry run, no effect `-t` display messages showing steps of cvs activity ------------------------------------------------------ ==== Latest files from CVS To get the latest files from CVS, use "`tomorrow`" by the following. -------------------- $ cvs ex -D tomorrow module_name -------------------- ==== Administration of CVS Add module alias "`mx`" to a CVS project (local server) by the following. -------------------- $ export CVSROOT=/srv/cvs/project $ cvs co CVSROOT/modules $ cd CVSROOT $ echo "mx -a module1" >>modules $ cvs ci -m "Now mx is an alias for module1" $ cvs release -d . -------------------- Now, you can check out "`module1`" (alias: "`mx`") from CVS to "`new`" directory by the following. -------------------- $ cvs co -d new mx $ cd new -------------------- NOTE: In order to perform above procedure, you should have appropriate file permissions. ==== Execution bit for CVS checkout When you checkout files from CVS, their execution permission bit is retained. Whenever you see execution permission problems in a checked out file, e.g. "`filename`", change its permission in the corresponding CVS repository by the following to fix it. -------------------- # chmod ugo-x filename -------------------- === Subversion Subversion is a **recent-generation** version control system replacing older CVS. It has most of CVS@@@sq@@@s features except tags and branches. You need to install `subversion`, `libapache2-svn` and `subversion-tools` packages to set up a Subversion server. ==== Configuration of Subversion repository Currently, the `subversion` package does not set up a repository, so one must set it up manually. One possible location for a repository is in "`/srv/svn/project`". Create a directory by the following. -------------------- # mkdir -p /srv/svn/project -------------------- Create the repository database by the following. -------------------- # svnadmin create /srv/svn/project -------------------- ==== Access to Subversion via Apache2 server If you only access Subversion repository via Apache2 server, you just need to make the repository only writable by the WWW server by the following. -------------------- # chown -R www-data:www-data /srv/svn/project -------------------- Add (or uncomment) the following in "`/etc/apache2/mods-available/dav_svn.conf`" to allow access to the repository via user authentication. -------------------- DAV svn SVNPath /srv/svn/project AuthType Basic AuthName "Subversion repository" AuthUserFile /etc/subversion/passwd Require valid-user -------------------- Create a user authentication file with the command by the following. -------------------- # htpasswd2 -c /etc/subversion/passwd some-username -------------------- Restart Apache2. Your new Subversion repository is accessible at URL "`http://localhost/project`" and "`http://example.com/project`" from `svn`(1) (assuming your URL of web server is "`http://example.com/`"). ==== Local access to Subversion by group The following sets up Subversion repository for the local access by a group, e.g. `project`. -------------------- # chmod 2775 /srv/svn/project # chown -R root:src /srv/svn/project # chmod -R ug+rwX /srv/svn/project -------------------- Your new Subversion repository is group accessible at URL "`file:@@@slash@@@//localhost/srv/svn/project`" or "`file:@@@slash@@@//srv/svn/project`" from `svn`(1) for local users belonging to `project` group. You must run commands, such as `svn`, `svnserve`, `svnlook`, and `svnadmin` under "`umask 002`" to ensure group access. ==== Remote access to Subversion via SSH A group accessible Subversion repository is at URL "`example.com:/srv/svn/project`" for SSH, you can access it from `svn`(1) at URL "`svn+ssh://example.com:/srv/svn/project`". ==== Subversion directory structure Many projects uses directory tree similar to the following for Subversion to compensate its lack of branches and tags. ---------------------------------------- ----- module1 | |-- branches | |-- tags | | |-- release-1.0 | | `-- release-2.0 | | | `-- trunk | |-- file1 | |-- file2 | `-- file3 | `-- module2 ---------------------------------------- TIP: You must use "`svn copy ...`" command to mark branches and tags. This ensures Subversion to record modification history of files properly and saves storage spaces. ==== Importing a new source to Subversion Create a new local source tree location at "`~/path/to/module1`" by the following. -------------------- $ mkdir -p ~/path/to/module1; cd ~/path/to/module1 -------------------- Populate a new local source tree under "`~/path/to/module1`" with files. Import it to Subversion with the following parameters. - Module name: "`module1`" - Subversion site URL: "`file:@@@slash@@@//srv/svn/project`" - Subversion directory: "`module1/trunk`" - Subversion tag: "`module1/tags/Release-initial`" -------------------- $ cd ~/path/to/module1 $ svn import file:///srv/svn/project/module1/trunk -m "Start module1" $ svn cp file:///srv/svn/project/module1/trunk file:///srv/svn/project/module1/tags/Release-initial -------------------- Alternatively, by the following. -------------------- $ svn import ~/path/to/module1 file:///srv/svn/project/module1/trunk -m "Start module1" $ svn cp file:///srv/svn/project/module1/trunk file:///srv/svn/project/module1/tags/Release-initial -------------------- TIP: You can replace URLs such as "`file:@@@slash@@@//...`" by any other URL formats such as "`http://...`" and "`svn+ssh://...`". ==== Work flow of Subversion Here is an example of typical work flow using Subversion with its native client. TIP: Client commands offered by the `git-svn` package may offer alternative work flow of Subversion using the `git` command. See <<_git_for_the_subversion_repository>>. Check all available modules from Subversion project pointed by URL "`file:///srv/svn/project`" by the following. -------------------- $ svn list file:///srv/svn/project module1 module2 ... -------------------- Checkout "`module1/trunk`" to a directory "`module1`" by the following. -------------------- $ cd ~/path/to $ svn co file:///srv/svn/project/module1/trunk module1 $ cd module1 -------------------- Make changes to the content as needed. Check changes by making "`diff -u [repository] [local]`" equivalent by the following. -------------------- $ svn diff -------------------- You find that you broke some file "`file_to_undo`" severely but other files are fine. Overwrite "`file_to_undo`" file with the clean copy from Subversion by the following. -------------------- $ svn revert file_to_undo -------------------- Save the updated local source tree to Subversion by the following. -------------------- $ svn ci -m "Describe change" -------------------- Create and add "`file_to_add`" file to Subversion by the following. -------------------- $ vi file_to_add $ svn add file_to_add $ svn ci -m "Added file_to_add" -------------------- Merge the latest version from Subversion by the following. -------------------- $ svn up -------------------- Watch out for lines starting with "`C filename`" which indicates conflicting changes. Look for unmodified code in, e.g., "`filename.r6`", "`filename.r9`", and "`filename.mine`". Search for "`<<<<<<<`" and "`>>>>>>>`" in files for conflicting changes. Edit files to fix conflicts as needed. Add a release tag "`Release-1`" by the following. -------------------- $ svn ci -m "last commit for Release-1" $ svn cp file:///srv/svn/project/module1/trunk file:///srv/svn/project/module1/tags/Release-1 -------------------- Edit further. Remove the release tag "`Release-1`" by the following. -------------------- $ svn rm file:///srv/svn/project/module1/tags/Release-1 -------------------- Check in changes to Subversion by the following. -------------------- $ svn ci -m "real last commit for Release-1" -------------------- Re-add the release tag "`Release-1`" from updated Subversion HEAD of trunk by the following. -------------------- $ svn cp file:///srv/svn/project/module1/trunk file:///srv/svn/project/module1/tags/Release-1 -------------------- Create a branch with a path "`module1/branches/Release-initial-bugfixes`" from the original version pointed by the path "`module1/tags/Release-initial`" and check it out to "`~/path/to/old`" directory by the following. -------------------- $ svn cp file:///srv/svn/project/module1/tags/Release-initial file:///srv/svn/project/module1/branches/Release-initial-bugfixes $ cd ~/path/to $ svn co file:///srv/svn/project/module1/branches/Release-initial-bugfixes old $ cd old -------------------- TIP: Use "`module1/trunk@\{2005-12-20\}`" (http://en.wikipedia.org/wiki/ISO_8601[ISO 8601] date format) instead of "`module1/tags/Release-initial`" to specify particular date as the branch point. Work on this local source tree pointing to branch "`Release-initial-bugfixes`" which is based on the original version. Work on this branch by yourself ... until someone else joins to this "`Release-initial-bugfixes`" branch. Sync with files modified by others on this branch by the following. -------------------- $ svn up -------------------- Edit files to fix conflicts as needed. Check in changes to Subversion by the following. -------------------- $ svn ci -m "checked into this branch" -------------------- Update the local tree with HEAD of trunk by the following. -------------------- $ svn switch file:///srv/svn/project/module1/trunk -------------------- Update the local tree (content = HEAD of trunk) by merging from the "`Release-initial-bugfixes`" branch by the following. -------------------- $ svn merge file:///srv/svn/project/module1/branches/Release-initial-bugfixes -------------------- Fix conflicts with editor. Check in changes to Subversion by the following. -------------------- $ svn ci -m "merged Release-initial-bugfixes" -------------------- Make archive by the following. -------------------- $ cd .. $ mv old old-module1-bugfixes $ tar -cvzf old-module1-bugfixes.tar.gz old-module1-bugfixes $ rm -rf old-module1-bugfixes -------------------- TIP: You can replace URLs such as "`file:@@@slash@@@//...`" by any other URL formats such as "`http://...`" and "`svn+ssh://...`". TIP: You can checkout only a sub directory of "`module1`" by providing its name as "`svn co file:@@@slash@@@//srv/svn/project/module1/trunk/subdir module1/subdir`", etc. .Notable options for Subversion commands (use as first argument(s) to `svn`(1)) [grid="all"] `------------`----------------------------------------- option meaning ------------------------------------------------------- `--dry-run` dry run, no effect `-v` display detail messages of svn activity ------------------------------------------------------- debian-reference-2.53/asciidoc/01_tutorial.txt0000644000000000000000000031232512255345173016206 0ustar == GNU/Linux tutorials // vim: set sts=2 expandtab: I think learning a computer system is like learning a new foreign language. Although tutorial books and documentation are helpful, you have to practice it yourself. In order to help you get started smoothly, I elaborate a few basic points. The powerful design of http://www.debian.org[Debian] http://en.wikipedia.org/wiki/GNU[GNU]/http://en.wikipedia.org/wiki/Linux[Linux] comes from the http://en.wikipedia.org/wiki/Unix[Unix] operating system, i.e., a http://en.wikipedia.org/wiki/Multi-user[multiuser], http://en.wikipedia.org/wiki/Computer_multitasking[multitasking] operating system. You must learn to take advantage of the power of these features and similarities between Unix and GNU/Linux. Don't shy away from Unix oriented texts and don't rely solely on GNU/Linux texts, as this robs you of much useful information. NOTE: If you have been using any http://en.wikipedia.org/wiki/Unix-like[Unix-like] system for a while with command line tools, you probably know everything I explain here. Please use this as a reality check and refresher. === Console basics ==== The shell prompt Upon starting the system, you are presented with the character based login screen if you did not install http://en.wikipedia.org/wiki/X_Window_System[X Window System] with the display manager such as `gdm3`. Suppose your hostname is `foo`, the login prompt looks as follows. -------------------- foo login: -------------------- If you did install a http://en.wikipedia.org/wiki/Graphical_user_interface[GUI] environment such as http://en.wikipedia.org/wiki/GNOME[GNOME] or http://en.wikipedia.org/wiki/KDE[KDE], then you can get to a login prompt by Ctrl-Alt-F1, and you can return to the GUI environment via Alt-F7 (see <<_virtual_consoles>> below for more). At the login prompt, you type your username, e.g. `penguin`, and press the Enter-key, then type your password and press the Enter-key again. NOTE: Following the Unix tradition, the username and password of the Debian system are case sensitive. The username is usually chosen only from the lowercase. The first user account is usually created during the installation. Additional user accounts can be created with `adduser`(8) by root. The system starts with the greeting message stored in "`/etc/motd`" (Message Of The Day) and presents a command prompt. -------------------- Debian GNU/Linux jessie/sid foo tty1 foo login: penguin Password: Last login: Mon Sep 23 19:36:44 JST 2013 on tty3 Linux snoopy 3.11-1-amd64 #1 SMP Debian 3.11.6-2 (2013-11-01) x86_64 The programs included with the Debian GNU/Linux system are free software; the exact distribution terms for each program are described in the individual files in /usr/share/doc/*/copyright. Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent permitted by applicable law. foo:~$ -------------------- Here, the main part of the greeting message can be customized by editing the "`/etc/motd.tail`" file. The first line is generated from the system information using "`uname -snrvm`". Now you are in the http://en.wikipedia.org/wiki/Shell_(computing)[shell]. The shell interprets your commands. ==== The shell prompt under X If you installed http://en.wikipedia.org/wiki/X_Window_System[X Window System] with a display manager such as http://en.wikipedia.org/wiki/GNOME[GNOME]@@@sq@@@s `gdm3` by selecting "Desktop environment" task during the installation, you are presented with the graphical login screen upon starting your system. You type your username and your password to login to the non-privileged user account. Use tab to navigate between username and password, or use the mouse and primary click. You can gain the shell prompt under X by starting a `x-terminal-emulator` program such as `gnome-terminal`(1), `rxvt`(1) or `xterm`(1). Under the GNOME Desktop environment, clicking "Applications" -> "Accessories" -> "Terminal" does the trick. You can also see the section below <<_virtual_consoles>>. Under some other Desktop systems (like `fluxbox`), there may be no obvious starting point for the menu. If this happens, just try (right) clicking the background of the desktop screen and hope for a menu to pop-up. ==== The root account The root account is also called http://en.wikipedia.org/wiki/Superuser[superuser] or privileged user. From this account, you can perform the following system administration tasks. - Read, write, and remove any files on the system irrespective of their file permissions - Set file ownership and permissions of any files on the system - Set the password of any non-privileged users on the system - Login to any accounts without their passwords This unlimited power of root account requires you to be considerate and responsible when using it. WARNING: Never share the root password with others. NOTE: File permissions of a file (including hardware devices such as CD-ROM etc. which are just another file for the Debian system) may render it unusable or inaccessible by non-root users. Although the use of root account is a quick way to test this kind of situation, its resolution should be done through proper setting of file permissions and user@@@sq@@@s group membership (see <<_filesystem_permissions>>). ==== The root shell prompt Here are a few basic methods to gain the root shell prompt by using the root password. - Type `root` at the character based login prompt. - Click "Applications" -> "Accessories" -> "Root Terminal", under the GNOME Desktop environment. - Type "`su -l`" from any user shell prompt. * This does not preserve the environment of the current user. - Type "`su`" from any user shell prompt. * This preserves some of the environment of the current user. ==== GUI system administration tools When your desktop menu does not start GUI system administration tools automatically with the appropriate privilege, you can start them from the root shell prompt of the X terminal emulator, such as `gnome-terminal`(1), `rxvt`(1), or `xterm`(1). See <<_the_root_shell_prompt>> and <<_running_x_clients_as_root>>. WARNING: Never start the X display/session manager under the root account by typing in `root` to the prompt of the display manager such as `gdm3`(1). WARNING: Never run untrusted remote GUI program under X Window when critical information is displayed since it may eavesdrop your X screen. // There is no xwindows. Please do not use such word here. // In some cases, it is necessary to run gui tools as root. The trick here is to {{{su}}} in {{{gnome-terminal}}}, {{{rxvt}}}, or {{{xterm}}} without the -l option. Then start the gui application by typing its name in the terminal that you {{{su}}}'d in. This starts the gui application as root, but preserves the environment of the user you are logged in as, so that things, like X Window functionality, don't break. ==== Virtual consoles In the default Debian system, there are six switchable http://en.wikipedia.org/wiki/VT100[VT100-like] character consoles available to start the command shell directly on the Linux host. Unless you are in a GUI environment, you can switch between the virtual consoles by pressing the `Left-Alt-key` and one of the `F1` -- `F6` keys simultaneously. Each character console allows independent login to the account and offers the multiuser environment. This multiuser environment is a great Unix feature, and very addictive. If you are under the X Window System, you gain access to the character console 1 by pressing `Ctrl-Alt-F1` key, i.e., the `left-Ctrl-key`, the `left-Alt-key`, and the `F1-key` are pressed together. You can get back to the X Window System, normally running on the virtual console 7, by pressing `Alt-F7`. You can alternatively change to another virtual console, e.g. to the console 1, from the commandline. -------------------- # chvt 1 -------------------- ==== How to leave the command prompt You type `Ctrl-D`, i.e., the `left-Ctrl-key` and the `d-key` pressed together, at the command prompt to close the shell activity. If you are at the character console, you return to the login prompt with this. Even though these control characters are referred as "control D" with the upper case, you do not need to press the Shift-key. The short hand expression, `\^D`, is also used for `Ctrl-D`. Alternately, you can type "exit". If you are at `x-terminal-emulator`(1), you can close `x-terminal-emulator` window with this. ==== How to shutdown the system Just like any other modern OS where the file operation involves http://en.wikipedia.org/wiki/Cache[caching data] in memory for improved performance, the Debian system needs the proper shutdown procedure before power can safely be turned off. This is to maintain the integrity of files, by forcing all changes in memory to be written to disk. If the software power control is available, the shutdown procedure automatically turns off power of the system. (Otherwise, you may have to press power button for few seconds after the shutdown procedure.) You can shutdown the system under the normal multiuser mode from the commandline. -------------------- # shutdown -h now -------------------- You can shutdown the system under the single-user mode from the commandline. -------------------- # poweroff -i -f -------------------- Alternatively, you may type `Ctrl-Alt-Delete` (The `left-Ctrl-key`, the `left-Alt-Key`, and the `Delete` are pressed together) to shutdown if "`/etc/inittab`" contains "`ca:12345:ctrlaltdel:/sbin/shutdown -t1 -a -h now`" in it. See `inittab`(5) for details. See <<_how_to_shutdown_the_remote_system_on_ssh>>. ==== Recovering a sane console When the screen goes berserk after doing some funny things such as "`cat `", type "`reset`" at the command prompt. You may not be able to see the command echoed as you type. You may also issue "`clear`" to clean up the screen. ==== Additional package suggestions for the newbie Although even the minimal installation of the Debian system without any desktop environment tasks provides the basic Unix functionality, it is a good idea to install few additional commandline and curses based character terminal packages such as `mc` and `vim` with `apt-get`(8) for beginners to get started by the following. -------------------- # apt-get update ... # apt-get install mc vim sudo ... -------------------- // Please do not add your favorite editor or any other software here without discussing debian-doc@lists.debian.org If you already had these packages installed, no new packages are installed. .List of interesting text-mode program packages [grid="all"] `----------`-------------`------------`-------------------------------------------------------------------------- package popcon size description ----------------------------------------------------------------------------------------------------------------- `mc` @-@popcon1@-@ @-@psize1@-@ A text-mode full-screen file manager `sudo` @-@popcon1@-@ @-@psize1@-@ A program to allow limited root privileges to users `vim` @-@popcon1@-@ @-@psize1@-@ Unix text editor Vi IMproved, a programmers text editor (standard version) `vim-tiny` @-@popcon1@-@ @-@psize1@-@ Unix text editor Vi IMproved, a programmers text editor (compact version) `emacs23` @-@popcon1@-@ @-@psize1@-@ GNU project Emacs, the Lisp based extensible text editor (version 23) `w3m` @-@popcon1@-@ @-@psize1@-@ Text-mode WWW browsers `gpm` @-@popcon1@-@ @-@psize1@-@ The Unix style cut-and-paste on the text console (daemon) ----------------------------------------------------------------------------------------------------------------- It may be a good idea to read some informative documentations. .List of informative documentation packages [grid="all"] `----------------------`-------------`------------`-------------------------------------------------------------- package popcon size description ----------------------------------------------------------------------------------------------------------------- `doc-debian` @-@popcon1@-@ @-@psize1@-@ Debian Project documentation, (Debian FAQ) and other documents `debian-policy` @-@popcon1@-@ @-@psize1@-@ Debian Policy Manual and related documents `developers-reference` @-@popcon1@-@ @-@psize1@-@ Guidelines and information for Debian developers `maint-guide` @-@popcon1@-@ @-@psize1@-@ Debian New Maintainers' Guide `debian-history` @-@popcon1@-@ @-@psize1@-@ History of the Debian Project `debian-faq` @-@popcon1@-@ @-@psize1@-@ Debian FAQ `sysadmin-guide` @-@popcon1@-@ @-@psize1@-@ The Linux System Administrators' Guide ----------------------------------------------------------------------------------------------------------------- You can install some of these packages by the following. -------------------- # apt-get install package_name -------------------- ==== An extra user account If you do not want to use your main user account for the following training activities, you can create a training user account, e.g. `fish` by the following. -------------------- # adduser fish -------------------- Answer all questions. This creates a new account named as `fish`. After your practice, you can remove this user account and its home directory by the following. -------------------- # deluser --remove-home fish -------------------- ==== sudo configuration For the typical single user workstation such as the desktop Debian system on the laptop PC, it is common to deploy simple configuration of `sudo`(8) as follows to let the non-privileged user, e.g. `penguin`, to gain administrative privilege just with his user password but without the root password. -------------------- # echo "penguin ALL=(ALL) ALL" >> /etc/sudoers -------------------- Alternatively, it is also common to do as follows to let the non-privileged user, e.g. `penguin`, to gain administrative privilege without any password. -------------------- # echo "penguin ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers -------------------- This trick should only be used for the single user workstation which you administer and where you are the only user. WARNING: Do not set up accounts of regular users on multiuser workstation like this because it would be very bad for system security. CAUTION: The password and the account of the `penguin` in the above example requires as much protection as the root password and the root account. // The following is less abrasive than one proposed. CAUTION: Administrative privilege in this context belongs to someone authorized to perform the system administration task on the workstation. Never give some manager in the Admin department of your company or your boss such privilege unless they are authorized and capable. NOTE: For providing access privilege to limited devices and limited files, you should consider to use **group** to provide limited access instead of using the `root` privilege via `sudo`(8). NOTE: With more thoughtful and careful configuration, `sudo`(8) can grant limited administrative privileges to other users on a shared system without sharing the root password. This can help with accountability with hosts with multiple administrators so you can tell who did what. On the other hand, you might not want anyone else to have such privileges. ==== Play time Now you are ready to play with the Debian system without risks as long as you use the non-privileged user account. This is because the Debian system is, even after the default installation, configured with proper file permissions which prevent non-privileged users from damaging the system. Of course, there may still be some holes which can be exploited but those who worry about these issues should not be reading this section but should be reading http://www.debian.org/doc/manuals/securing-debian-howto/[Securing Debian Manual]. // titles are noun phrases We learn the Debian system as a http://en.wikipedia.org/wiki/Unix-like[Unix-like] system with the following. - <<_unix_like_filesystem>> (basic concept) - <<_midnight_commander_mc>> (survival method) - <<_the_basic_unix_like_work_environment>> (basic method) - <<_the_simple_shell_command>> (shell mechanism) - <<_unix_like_text_processing>> (text processing method) === Unix-like filesystem In GNU/Linux and other http://en.wikipedia.org/wiki/Unix-like[Unix-like] operating systems, http://en.wikipedia.org/wiki/Computer_file[files] are organized into http://en.wikipedia.org/wiki/Directory_(file_systems)[directories]. All files and directories are arranged in one big tree rooted at "`/`". It@@@sq@@@s called a tree because if you draw the filesystem, it looks like a tree but it is upside down. These files and directories can be spread out over several devices. `mount`(8) serves to attach the filesystem found on some device to the big file tree. Conversely, `umount`(8) detaches it again. On recent Linux kernels, `mount`(8) with some options can bind part of a file tree somewhere else or can mount filesystem as shared, private, slave, or unbindable. Supported mount options for each filesystem are available in "`/share/doc/linux-doc-\*/Documentation/filesystems/`". **Directories** on Unix systems are called **folders** on some other systems. Please also note that there is no concept for **drive** such as "`A:`" on any Unix system. There is one filesystem, and everything is included. This is a huge advantage compared to Windows. ==== Unix file basics Here are some Unix file basics. - Filenames are **case sensitive**. That is, "`MYFILE`" and "`MyFile`" are different files. - The **root directory** means root of the filesystem referred as simply "`/`". Don't confuse this with the home directory for the root user: "`/root`". - Every directory has a name which can contain any letters or symbols **except "`/`"**. The root directory is an exception; its name is "`/`" (pronounced "slash" or "the root directory") and it cannot be renamed. - Each file or directory is designated by a **fully-qualified filename**, **absolute filename**, or **path**, giving the sequence of directories which must be passed through to reach it. The three terms are synonymous. - All **fully-qualified filenames** begin with the "`/`" directory, and there@@@sq@@@s a "`/`" between each directory or file in the filename. The first "`/`" is the top level directory, and the other "`/`"@@@sq@@@s separate successive subdirectories, until we reach the last entry which is the name of the actual file. The words used here can be confusing. Take the following **fully-qualified filename** as an example: "`/usr/share/keytables/us.map.gz`". However, people also refers to its basename "`us.map.gz`" alone as a filename. - The root directory has a number of branches, such as "`/etc/`" and "`/usr/`". These subdirectories in turn branch into still more subdirectories, such as "`/etc/init.d/`" and "`/usr/local/`". The whole thing viewed collectively is called the **directory tree**. You can think of an absolute filename as a route from the base of the tree ("`/`") to the end of some branch (a file). You also hear people talk about the directory tree as if it were a **family** tree encompassing all direct descendants of a single figure called the root directory ("`/`"): thus subdirectories have **parents**, and a path shows the complete ancestry of a file. There are also relative paths that begin somewhere other than the root directory. You should remember that the directory "`../`" refers to the parent directory. This terminology also applies to other directory like structures, such as hierarchical data structures. - There@@@sq@@@s no special directory path name component that corresponds to a physical device, such as your hard disk. This differs from http://en.wikipedia.org/wiki/RT-11[RT-11], http://en.wikipedia.org/wiki/CP/M[CP/M], http://en.wikipedia.org/wiki/OpenVMS[OpenVMS], http://en.wikipedia.org/wiki/MS-DOS[MS-DOS], http://en.wikipedia.org/wiki/AmigaOS[AmigaOS], and http://en.wikipedia.org/wiki/Microsoft_Windows[Microsoft Windows], where the path contains a device name such as "`C:\`". (However, directory entries do exist that refer to physical devices as a part of the normal filesystem. See <<_filesystem_internals>>.) NOTE: While you **can** use almost any letters or symbols in a file name, in practice it is a bad idea to do so. It is better to avoid any characters that often have special meanings on the command line, including spaces, tabs, newlines, and other special characters: `{ } ( ) [ ] @@@sq@@@ @@@grave@@@ @@@dq@@@ \ @@@slash@@@ > < | ; ! # @@@amp@@@ @@@hat@@@ @@@star@@@ % @ $` . If you want to separate words in a name, good choices are the period, hyphen, and underscore. You could also capitalize each word, "`LikeThis`". Experienced Linux users tend to avoid spaces in filenames. NOTE: The word "root" can mean either "root user" or "root directory". The context of their usage should make it clear. NOTE: The word **path** is used not only for **fully-qualified filename** as above but also for the **command search path**. The intended meaning is usually clear from the context. The detailed best practices for the file hierarchy are described in the Filesystem Hierarchy Standard ("`/usr/share/doc/debian-policy/fhs/fhs-2.3.txt.gz`" and `hier`(7)). You should remember the following facts as the starter. .List of usage of key directories [grid="all"] `-----------`------------------------------------------------------------------------------------------- directory usage of the directory -------------------------------------------------------------------------------------------------------- `/` the root directory `/etc/` system wide configuration files `/var/log/` system log files `/home/` all the home directories for all non-privileged users -------------------------------------------------------------------------------------------------------- ==== Filesystem internals Following the **Unix tradition**, the Debian GNU/Linux system provides the http://en.wikipedia.org/wiki/File_system[filesystem] under which physical data on hard disks and other storage devices reside, and the interaction with the hardware devices such as console screens and remote serial consoles are represented in an unified manner under "`/dev/`". Each file, directory, named pipe (a way two programs can share data), or physical device on a Debian GNU/Linux system has a data structure called an http://en.wikipedia.org/wiki/Inode[inode] which describes its associated attributes such as the user who owns it (owner), the group that it belongs to, the time last accessed, etc. If you are really interested, see "`/usr/include/linux/fs.h`" for the exact definition of "`struct inode`" in the Debian GNU/Linux system. The idea of representing just about everything in the filesystem was a Unix innovation, and modern Linux kernels have developed this idea ever further. Now, even information about processes running in the computer can be found in the filesystem. This abstract and unified representation of physical entities and internal processes is very powerful since this allows us to use the same command for the same kind of operation on many totally different devices. It is even possible to change the way the kernel works by writing data to special files that are linked to running processes. // I am commenting out since redundant and long, no advocacy please. // Please do not deviates from focus topic. Just keep this as technical information!!! // I start this section as '''Unix tradition''' : /!\ Reread that last sentence. This is the essence of an incredible amount of flexibility, power, and utility that is at the heart of Unix/Linux based systems. This type of power is not easily tapped from the Graphical User Interface, and this is why if you administer, program, or use Unix/Linux based systems, it really calls for you to learn the power of the command line and why you benefit from knowing how your system works. -- But most of tweaking requires C programing though to get really into details. // All of your files could be on one disk --- or you could have 20 disks, some of them connected to a different computer elsewhere on the network. You can't tell just by looking at the directory tree, and nearly all commands work just the same way no matter what physical device(s) your files are really on. [This is a good thing. Trust us.] Of course, methods do exist whereby you can tell what devices map to which physical or network devices. {{{mount}}} with no arguments shows how storage is mapped to physical or network devices. TIP: If you need to identify the correspondence between the file tree and the physical entity, execute `mount`(8) with no arguments. ==== Filesystem permissions http://en.wikipedia.org/wiki/File_system_permissions[Filesystem permissions] of http://en.wikipedia.org/wiki/Unix-like[Unix-like] system are defined for three categories of affected users. - The **user** who owns the file (**u**) - Other users in the **group** which the file belongs to (**g**) - All **other** users (**o**) also referred to as "world" and "everyone" For the file, each corresponding permission allows following actions. - The **read** (**r**) permission allows owner to examine contents of the file. - The **write** (**w**) permission allows owner to modify the file. - The **execute** (**x**) permission allows owner to run the file as a command. For the directory, each corresponding permission allows following actions. - The **read** (**r**) permission allows owner to list contents of the directory. - The **write** (**w**) permission allows owner to add or remove files in the directory. - The **execute** (**x**) permission allows owner to access files in the directory. Here, the **execute** permission on a directory means not only to allow reading of files in that directory but also to allow viewing their attributes, such as the size and the modification time. `ls`(1) is used to display permission information (and more) for files and directories. When it is invoked with the "`-l`" option, it displays the following information in the order given. - **Type of file** (first character) - Access **permission** of the file (nine characters, consisting of three characters each for user, group, and other in this order) - **Number of hard links** to the file - Name of the **user** who owns the file - Name of the **group** which the file belongs to - **Size** of the file in characters (bytes) - **Date and time** of the file (mtime) - **Name** of the file .List of the first character of "`ls -l`" output [grid="all"] `---------`--------------------- character meaning -------------------------------- `-` normal file `d` directory `l` symlink `c` character device node `b` block device node `p` named pipe `s` socket -------------------------------- `chown`(1) is used from the root account to change the owner of the file. `chgrp`(1) is used from the file@@@sq@@@s owner or root account to change the group of the file. `chmod`(1) is used from the file@@@sq@@@s owner or root account to change file and directory access permissions. Basic syntax to manipulate a `foo` file is the following. -------------------- # chown foo # chgrp foo # chmod [ugoa][+-=][rwxXst][,...] foo -------------------- For example, you can make a directory tree to be owned by a user `foo` and shared by a group `bar` by the following. -------------------- # cd /some/location/ # chown -R foo:bar . # chmod -R ug+rwX,o=rX . -------------------- There are three more special permission bits. - The **set user ID** bit (**s** or **S** instead of user@@@sq@@@s **x**) - The **set group ID** bit (**s** or **S** instead of group@@@sq@@@s **x**) - The **sticky** bit (**t** or **T** instead of other@@@sq@@@s **x**) Here the output of "`ls -l`" for these bits is **capitalized** if execution bits hidden by these outputs are **unset**. Setting **set user ID** on an executable file allows a user to execute the executable file with the owner ID of the file (for example **root**). Similarly, setting **set group ID** on an executable file allows a user to execute the executable file with the group ID of the file (for example **root**). Because these settings can cause security risks, enabling them requires extra caution. Setting **set group ID** on a directory enables the http://en.wikipedia.org/wiki/Berkeley_Software_Distribution[BSD-like] file creation scheme where all files created in the directory belong to the **group** of the directory. Setting the **sticky bit** on a directory prevents a file in the directory from being removed by a user who is not the owner of the file. In order to secure contents of a file in world-writable directories such as "`/tmp`" or in group-writable directories, one must not only reset the **write** permission for the file but also set the **sticky bit** on the directory. Otherwise, the file can be removed and a new file can be created with the same name by any user who has write access to the directory. Here are a few interesting examples of file permissions. -------------------- $ ls -l /etc/passwd /etc/shadow /dev/ppp /usr/sbin/exim4 crw------T 1 root root 108, 0 Oct 16 20:57 /dev/ppp -rw-r--r-- 1 root root 2761 Aug 30 10:38 /etc/passwd -rw-r----- 1 root shadow 1695 Aug 30 10:38 /etc/shadow -rwsr-xr-x 1 root root 973824 Sep 23 20:04 /usr/sbin/exim4 $ ls -ld /tmp /var/tmp /usr/local /var/mail /usr/src drwxrwxrwt 14 root root 20480 Oct 16 21:25 /tmp drwxrwsr-x 10 root staff 4096 Sep 29 22:50 /usr/local drwxr-xr-x 10 root root 4096 Oct 11 00:28 /usr/src drwxrwsr-x 2 root mail 4096 Oct 15 21:40 /var/mail drwxrwxrwt 3 root root 4096 Oct 16 21:20 /var/tmp -------------------- There is an alternative numeric mode to describe file permissions with `chmod`(1). This numeric mode uses 3 to 4 digit wide octal (radix=8) numbers. .The numeric mode for file permissions in `chmod`(1) commands [grid="all"] `------------------`----------------------------------------------------------------------------------- digit meaning ------------------------------------------------------------------------------------------------------- 1st optional digit sum of **set user ID** (=4), **set group ID** (=2), and **sticky bit** (=1) 2nd digit sum of **read** (=4), **write** (=2), and **execute** (=1) permissions for **user** 3rd digit ditto for **group** 4th digit ditto for **other** ------------------------------------------------------------------------------------------------------- This sounds complicated but it is actually quite simple. If you look at the first few (2-10) columns from "`ls -l`" command output and read it as a binary (radix=2) representation of file permissions ("-" being "0" and "rwx" being "1"), the last 3 digit of the numeric mode value should make sense as an octal (radix=8) representation of file permissions to you. For example, try the following -------------------- $ touch foo bar $ chmod u=rw,go=r foo $ chmod 644 bar $ ls -l foo bar -rw-r--r-- 1 penguin penguin 0 Oct 16 21:39 bar -rw-r--r-- 1 penguin penguin 0 Oct 16 21:35 foo -------------------- TIP: If you need to access information displayed by "`ls -l`" in shell script, you should use pertinent commands such as `test`(1), `stat`(1) and `readlink`(1). The shell builtin such as "`[`" or "`test`" may be used too. ==== Control of permissions for newly created files: umask What permissions are applied to a newly created file or directory is restricted by the `umask` shell builtin command. See `dash`(1), `bash`(1), and `builtins`(7). -------------------- (file permissions) = (requested file permissions) & ~(umask value) -------------------- .The **umask** value examples [grid="all"] `------`------------------------`-----------------------------`------------------------- umask file permissions created directory permissions created usage ---------------------------------------------------------------------------------------- `0022` `-rw-r--r--` `-rwxr-xr-x` writable only by the user `0002` `-rw-rw-r--` `-rwxrwxr-x` writable by the group ---------------------------------------------------------------------------------------- The Debian system uses a user private group (UPG) scheme as its default. A UPG is created whenever a new user is added to the system. A UPG has the same name as the user for which it was created and that user is the only member of the UPG. UPG scheme makes it safe to set umask to `0002` since every user has their own private group. (In some Unix variants, it is quite common to setup all normal users belonging to a single **`users`** group and is a good idea to set umask to `0022` for security in such cases.) TIP: Enable UPG by putting "`umask 002`" in the `~/.bashrc` file. ==== Permissions for groups of users (group) In order to make group permissions to be applied to a particular user, that user needs to be made a member of the group using "`sudo vigr`" for `/etc/group` and "`sudo vigr -s`" for `/etc/gshadow`. You need to login after logout (or run "`exec newgrp`") to enable the new group configuration. NOTE: Alternatively, you may dynamically add users to groups during the authentication process by adding "`auth optional pam_group.so`" line to "`/etc/pam.d/common-auth`" and setting "`/etc/security/group.conf`". (See <<_authentication>>.) The hardware devices are just another kind of file on the Debian system. If you have problems accessing devices such as CD-ROM and USB memory stick from a user account, you should make that user a member of the relevant group. Some notable system-provided groups allow their members to access particular files and devices without `root` privilege. .List of notable system-provided groups for file access [grid="all"] `---------`----------------------------------------------------------------------------- group description for accessible files and devices ---------------------------------------------------------------------------------------- `dialout` full and direct access to serial ports ("`/dev/ttyS[0-3]`") `dip` limited access to serial ports for **Dialup IP** connection to trusted peers `cdrom` CD-ROM, DVD+/-RW drives `audio` audio device `video` video device `scanner` scanner(s) `adm` system monitoring logs `staff` some directories for junior administrative work: "`/usr/local`", "`/home`" ---------------------------------------------------------------------------------------- TIP: You need to belong to the `dialout` group to reconfigure modem, dial anywhere, etc. But if `root` creates pre-defined configuration files for trusted peers in "`/etc/ppp/peers/`", you only need to belong to the `dip` group to create **Dialup IP** connection to those trusted peers using `pppd`(8), `pon`(1), and `poff`(1) commands. Some notable system-provided groups allow their members to execute particular commands without `root` privilege. .List of notable system provided groups for particular command executions [grid="all"] `---------`---------------------------------------------------------------------------- group accessible commands --------------------------------------------------------------------------------------- `sudo` execute `sudo` without their password `lpadmin` execute commands to add, modify, and remove printers from printer databases --------------------------------------------------------------------------------------- For the full listing of the system provided users and groups, see the recent version of the "Users and Groups" document in "`/usr/share/doc/base-passwd/users-and-groups.html`" provided by the `base-passwd` package. See `passwd`(5), `group`(5), `shadow`(5), `newgrp`(1), `vipw`(8), `vigr`(8), and `pam_group`(8) for management commands of the user and group system. ==== Timestamps There are three types of timestamps for a GNU/Linux file. .List of types of timestamps [grid="all"] `---------`-------------------------------------- type meaning ------------------------------------------------- **mtime** the file modification time (`ls -l`) **ctime** the file status change time (`ls -lc`) **atime** the last file access time (`ls -lu`) ------------------------------------------------- NOTE: **ctime** is not file creation time. - Overwriting a file changes all of the **mtime**, **ctime**, and **atime** attributes of the file. - Changing ownership or permission of a file changes the **ctime** and **atime** attributes of the file. - Reading a file changes the **atime** of the file. NOTE: Even simply reading a file on the Debian system normally causes a file write operation to update **atime** information in the **inode**. Mounting a filesystem with "`noatime`" or "`relatime`" option makes the system skip this operation and results in faster file access for the read. This is often recommended for laptops, because it reduces hard drive activity and saves power. See `mount`(8). Use `touch`(1) command to change timestamps of existing files. For timestamps, the `ls` command outputs different strings under non-English locale ("`fr_FR.UTF-8`") from under the old one ("`C`"). -------------------- $ LANG=fr_FR.UTF-8 ls -l foo -rw-rw-r-- 1 penguin penguin 0 oct. 16 21:35 foo $ LANG=C ls -l foo -rw-rw-r-- 1 penguin penguin 0 Oct 16 21:35 foo -------------------- TIP: See <<_customized_display_of_time_and_date>> to customize "`ls -l`" output. ==== Links There are two methods of associating a file "`foo`" with a different filename "`bar`". - http://en.wikipedia.org/wiki/Hard_link[Hard link] * Duplicate name for an existing file * "`ln foo bar`" - http://en.wikipedia.org/wiki/Symbolic_link[Symbolic link or symlink] * Special file that points to another file by name * "`ln -s foo bar`" See the following example for changes in link counts and the subtle differences in the result of the `rm` command. -------------------- $ umask 002 $ echo "Original Content" > foo $ ls -li foo 1449840 -rw-rw-r-- 1 penguin penguin 17 Oct 16 21:42 foo $ ln foo bar # hard link $ ln -s foo baz # symlink $ ls -li foo bar baz 1449840 -rw-rw-r-- 2 penguin penguin 17 Oct 16 21:42 bar 1450180 lrwxrwxrwx 1 penguin penguin 3 Oct 16 21:47 baz -> foo 1449840 -rw-rw-r-- 2 penguin penguin 17 Oct 16 21:42 foo $ rm foo $ echo "New Content" > foo $ ls -li foo bar baz 1449840 -rw-rw-r-- 1 penguin penguin 17 Oct 16 21:42 bar 1450180 lrwxrwxrwx 1 penguin penguin 3 Oct 16 21:47 baz -> foo 1450183 -rw-rw-r-- 1 penguin penguin 12 Oct 16 21:48 foo $ cat bar Original Content $ cat baz New Content -------------------- The hardlink can be made within the same filesystem and shares the same inode number which the "`-i`" option with `ls`(1) reveals. The symlink always has nominal file access permissions of "`rwxrwxrwx`", as shown in the above example, with the effective access permissions dictated by permissions of the file that it points to. CAUTION: It is generally a good idea not to create complicated symbolic links or hardlinks at all unless you have a very good reason. It may cause nightmares where the logical combination of the symbolic links results in loops in the filesystem. NOTE: It is generally preferable to use symbolic links rather than hardlinks unless you have a good reason for using a hardlink. The "`.`" directory links to the directory that it appears in, thus the link count of any new directory starts at 2. The "`..`" directory links to the parent directory, thus the link count of the directory increases with the addition of new subdirectories. If you are just moving to Linux from Windows, it soon becomes clear how well-designed the filename linking of Unix is, compared with the nearest Windows equivalent of "shortcuts". Because it is implemented in the filesystem, applications can't see any difference between a linked file and the original. In the case of hardlinks, there really is no difference. ==== Named pipes (FIFOs) A http://en.wikipedia.org/wiki/Named_pipe[named pipe] is a file that acts like a pipe. You put something into the file, and it comes out the other end. Thus it@@@sq@@@s called a FIFO, or First-In-First-Out: the first thing you put in the pipe is the first thing to come out the other end. If you write to a named pipe, the process which is writing to the pipe doesn't terminate until the information being written is read from the pipe. If you read from a named pipe, the reading process waits until there is nothing to read before terminating. The size of the pipe is always zero --- it does not store data, it just links two processes like the functionality offered by the shell "`|`" syntax. However, since this pipe has a name, the two processes don't have to be on the same command line or even be run by the same user. Pipes were a very influential innovation of Unix. For example, try the following -------------------- $ cd; mkfifo mypipe $ echo "hello" >mypipe & # put into background [1] 8022 $ ls -l mypipe prw-rw-r-- 1 penguin penguin 0 Oct 16 21:49 mypipe $ cat mypipe hello [1]+ Done echo "hello" >mypipe $ ls mypipe mypipe $ rm mypipe -------------------- ==== Sockets Sockets are used extensively by all the Internet communication, databases, and the operating system itself. It is similar to the named pipe (FIFO) and allows processes to exchange information even between different computers. For the socket, those processes do not need to be running at the same time nor to be running as the children of the same ancestor process. This is the endpoint for http://en.wikipedia.org/wiki/Inter-process_communication[the inter process communication (IPC)]. The exchange of information may occur over the network between different hosts. The two most common ones are http://en.wikipedia.org/wiki/Internet_socket[the Internet socket] and http://en.wikipedia.org/wiki/Unix_domain_socket[the Unix domain socket]. TIP: "`netstat -an`" provides a very useful overview of sockets that are open on a given system. ==== Device files http://en.wikipedia.org/wiki/Device_file[Device files] refer to physical or virtual devices on your system, such as your hard disk, video card, screen, or keyboard. An example of a virtual device is the console, represented by "`/dev/console`". There are 2 types of device files. - **Character device** * Accessed one character at a time * 1 character = 1 byte * E.g. keyboard device, serial port, ... - **Block device** * accessed in larger units called blocks * 1 block > 1 byte * E.g. hard disk, ... You can read and write device files, though the file may well contain binary data which may be an incomprehensible-to-humans gibberish. Writing data directly to these files is sometimes useful for the troubleshooting of hardware connections. For example, you can dump a text file to the printer device "`/dev/lp0`" or send modem commands to the appropriate serial port "`/dev/ttyS0`". But, unless this is done carefully, it may cause a major disaster. So be cautious. NOTE: For the normal access to a printer, use `lp`(1). The device node number are displayed by executing `ls`(1) as the following. -------------------- $ ls -l /dev/sda /dev/sr0 /dev/ttyS0 /dev/zero brw-rw---T 1 root disk 8, 0 Oct 16 20:57 /dev/sda brw-rw---T+ 1 root cdrom 11, 0 Oct 16 21:53 /dev/sr0 crw-rw---T 1 root dialout 4, 64 Oct 16 20:57 /dev/ttyS0 crw-rw-rw- 1 root root 1, 5 Oct 16 20:57 /dev/zero -------------------- - "`/dev/sda`" has the major device number 8 and the minor device number 0. This is read/write accessible by users belonging to the `disk` group. - "`/dev/sr0`" has the major device number 11 and the minor device number 0. This is read/write accessible by users belonging to the `cdrom` group. - "`/dev/ttyS0`" has the major device number 4 and the minor device number 64. This is read/write accessible by users belonging to the `dialout` group. - "`/dev/zero`" has the major device number 1 and the minor device number 5. This is read/write accessible by anyone. On the modern Linux system, the filesystem under "`/dev/`" is automatically populated by the `udev`(7) mechanism. ==== Special device files There are some special device files. .List of special device files [grid="all"] `--------------`------`------------------------------------------------------------------------------------------------- device file action description of response ------------------------------------------------------------------------------------------------------------------------ `/dev/null` read return "end-of-file (EOF) character" `/dev/null` write return nothing (a bottomless data dump pit) `/dev/zero` read return "the `\0` (NUL) character" (not the same as the number zero ASCII) `/dev/random` read return random characters from a true random number generator, delivering real entropy (slow) `/dev/urandom` read return random characters from a cryptographically secure pseudorandom number generator `/dev/full` write return the disk-full (ENOSPC) error ------------------------------------------------------------------------------------------------------------------------ These are frequently used in conjunction with the shell redirection (see <<_typical_command_sequences_and_shell_redirection>>). ==== procfs and sysfs The http://en.wikipedia.org/wiki/Procfs[procfs] and http://en.wikipedia.org/wiki/Sysfs[sysfs] mounted on "`/proc`" and "`/sys`" are the pseudo-filesystem and expose internal data structures of the kernel to the userspace. In other word, these entries are virtual, meaning that they act as a convenient window into the operation of the operating system. The directory "`/proc`" contains (among other things) one subdirectory for each process running on the system, which is named after the process ID (PID). System utilities that access process information, such as `ps`(1), get their information from this directory structure. The directories under "`/proc/sys/`" contain interfaces to change certain kernel parameters at run time. (You may do the same through the specialized `sysctl`(8) command or its preload/configuration file "`/etc/sysctl.conf`".) People frequently panic when they notice one file in particular - "`/proc/kcore`" - which is generally huge. This is (more or less) a copy of the content of your computer@@@sq@@@s memory. It@@@sq@@@s used to debug the kernel. It is a virtual file that points to computer memory, so don't worry about its size. The directory under "`/sys`" contains exported kernel data structures, their attributes, and their linkages between them. It also contains interfaces to change certain kernel parameters at run time. See "`proc.txt(.gz)`", "`sysfs.txt(.gz)`" and other related documents in the Linux kernel documentation ("`/usr/share/doc/linux-doc-\*/Documentation/filesystems/\*`") provided by the `linux-doc-\*` package. ==== tmpfs The http://en.wikipedia.org/wiki/Tmpfs#Linux[tmpfs] is a temporary filesystem which keeps all files in the http://en.wikipedia.org/wiki/Virtual_memory[virtual memory]. The data of the tmpfs in the http://en.wikipedia.org/wiki/Page_cache[page cache] on memory may be swapped out to the http://en.wikipedia.org/wiki/Paging[swap space] on disk as needed. The directory "`/run`" is mounted as the tmpfs in the early boot process. This enables writing to it even when the directory "`/`" is mounted as read-only. This is the new location for the storage of transient state files and replaces several locations described in the http://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard[Filesystem Hierarchy Standard] version 2.3: - "`/var/run`" → "`/run`" - "`/var/lock`" → "`/run/lock`" - "`/dev/shm`" → "`/run/shm`" See "`tmpfs.txt(.gz)`" in the Linux kernel documentation ("`/usr/share/doc/linux-doc-\*/Documentation/filesystems/\*`") provided by the `linux-doc-\*` package. === Midnight Commander (MC) http://en.wikipedia.org/wiki/Midnight_Commander[Midnight Commander (MC)] is a GNU "Swiss army knife" for the Linux console and other terminal environments. This gives newbie a menu driven console experience which is much easier to learn than standard Unix commands. You may need to install the Midnight Commander package which is titled "`mc`" by the following. ---- $ sudo apt-get install mc ---- Use the `mc`(1) command to explore the Debian system. This is the best way to learn. Please explore few interesting locations just using the cursor keys and Enter key. - "`/etc`" and its subdirectories - "`/var/log`" and its subdirectories - "`/usr/share/doc`" and its subdirectories - "`/sbin`" and "`/bin`" ==== Customization of MC In order to make MC to change working directory upon exit and `cd` to the directory, I suggest to modify "`\~/.bashrc`" to include a script provided by the `mc` package. ---- . /usr/lib/mc/mc.sh ---- See `mc`(1) (under the "`-P`" option) for the reason. (If you do not understand what exactly I am talking here, you can do this later.) ==== Starting MC MC can be started by the following. -------------------- $ mc -------------------- MC takes care of all file operations through its menu, requiring minimal user effort. Just press F1 to get the help screen. You can play with MC just by pressing cursor-keys and function-keys. NOTE: In some consoles such as `gnome-terminal`(1), key strokes of function-keys may be stolen by the console program. You can disable these features by "Edit" -> "Keyboard Shortcuts" for `gnome-terminal`. If you encounter character encoding problem which displays garbage characters, adding "`-a`" to MC@@@sq@@@s command line may help prevent problems. If this doesn't clear up your display problems with MC, see <<_the_terminal_configuration>>. ==== File manager in MC The default is two directory panels containing file lists. Another useful mode is to set the right window to "information" to see file access privilege information, etc. Following are some essential keystrokes. With the `gpm`(8) daemon running, one can use a mouse on Linux character consoles, too. (Make sure to press the shift-key to obtain the normal behavior of cut and paste in MC.) .The key bindings of MC [grid="all"] `--------------------`----------------------------------------------------- key key binding --------------------------------------------------------------------------- `F1` help menu `F3` internal file viewer `F4` internal editor `F9` activate pull down menu `F10` exit Midnight Commander `Tab` move between two windows `Insert` or `Ctrl-T` mark file for a multiple-file operation such as copy `Del` delete file (be careful---set MC to safe delete mode) Cursor keys self-explanatory --------------------------------------------------------------------------- ==== Command-line tricks in MC - `cd` command changes the directory shown on the selected screen. - `Ctrl-Enter` or `Alt-Enter` copies a filename to the command line. Use this with `cp`(1) and `mv`(1) commands together with command-line editing. - `Alt-Tab` shows shell filename expansion choices. - One can specify the starting directory for both windows as arguments to MC; for example, "`mc /etc /root`". - `Esc` \+ `n-key` -> `Fn` (i.e., `Esc` \+ `1` -> `F1`, etc.; `Esc` \+ `0` -> `F10`) - Pressing `Esc` before the key has the same effect as pressing the `Alt` and the key together.; i.e., type `Esc` \+ `c` for `Alt-C`. `Esc` is called meta-key and sometimes noted as "`M-`". ==== The internal editor in MC The internal editor has an interesting cut-and-paste scheme. Pressing `F3` marks the start of a selection, a second `F3` marks the end of selection and highlights the selection. Then you can move your cursor. If you press F6, the selected area is moved to the cursor location. If you press F5, the selected area is copied and inserted at the cursor location. `F2` saves the file. `F10` gets you out. Most cursor keys work intuitively. This editor can be directly started on a file using one of the following commands. -------------------- $ mc -e filename_to_edit -------------------- -------------------- $ mcedit filename_to_edit -------------------- This is not a multi-window editor, but one can use multiple Linux consoles to achieve the same effect. To copy between windows, use Alt-F keys to switch virtual consoles and use "File->Insert file" or "File->Copy to file" to move a portion of a file to another file. This internal editor can be replaced with any external editor of choice. Also, many programs use the environment variables "`$EDITOR`" or "`$VISUAL`" to decide which editor to use. If you are uncomfortable with `vim`(1) or `nano`(1) initially, you may set these to "`mcedit`" by adding the following lines to "`\~/.bashrc`". -------------------- export EDITOR=mcedit export VISUAL=mcedit -------------------- I do recommend setting these to "`vim`" if possible. If you are uncomfortable with `vim`(1), you can keep using `mcedit`(1) for most system maintenance tasks. ==== The internal viewer in MC MC is a very smart viewer. This is a great tool for searching words in documents. I always use this for files in the "`/usr/share/doc`" directory. This is the fastest way to browse through masses of Linux information. This viewer can be directly started using one of the following commands. -------------------- $ mc -v path/to/filename_to_view -------------------- -------------------- $ mcview path/to/filename_to_view -------------------- ==== Auto-start features of MC Press Enter on a file, and the appropriate program handles the content of the file (see <<_customizing_program_to_be_started>>). This is a very convenient MC feature. .The reaction to the enter key in MC [grid="all"] `---------------------------------`-------------------------------------- file type reaction to enter key ------------------------------------------------------------------------- executable file execute command man file pipe content to viewer software html file pipe content to web browser "`\*.tar.gz`" and "`\*.deb`" file browse its contents as if subdirectory ------------------------------------------------------------------------- In order to allow these viewer and virtual file features to function, viewable files should not be set as executable. Change their status using `chmod`(1) or via the MC file menu. ==== FTP virtual filesystem of MC MC can be used to access files over the Internet using FTP. Go to the menu by pressing `F9`, then type "`p`" to activate the FTP virtual filesystem. Enter a URL in the form "`username:passwd@hostname.domainname`", which retrieves a remote directory that appears like a local one. Try "[http.us.debian.org/debian]" as the URL and browse the Debian archive. === The basic Unix-like work environment Although MC enables you to do almost everything, it is very important for you to learn how to use the command line tools invoked from the shell prompt and become familiar with the Unix-like work environment. ==== The login shell You can select your login shell with `chsh`(1). [[list-of-shell-programs]] .List of shell programs [grid="all"] `-------`-------------`------------`-----------`------------------------------------------------------------------------------------------ package popcon size POSIX shell description ------------------------------------------------------------------------------------------------------------------------------------------ `bash` @-@popcon1@-@ @-@psize1@-@ Yes http://en.wikipedia.org/wiki/Bash[Bash]: the GNU Bourne Again SHell (de facto standard) `tcsh` @-@popcon1@-@ @-@psize1@-@ No http://en.wikipedia.org/wiki/Tcsh[TENEX C Shell]: an enhanced version of http://en.wikipedia.org/wiki/C_shell[Berkeley csh] `dash` @-@popcon1@-@ @-@psize1@-@ Yes Debian http://en.wikipedia.org/wiki/Almquist_shell[Almquist Shell], good for shell script `zsh` @-@popcon1@-@ @-@psize1@-@ Yes http://en.wikipedia.org/wiki/Z_shell[Z shell]: the standard shell with many enhancements `pdksh` @-@popcon1@-@ @-@psize1@-@ Yes public domain version of the http://en.wikipedia.org/wiki/Korn_shell[Korn shell] `csh` @-@popcon1@-@ @-@psize1@-@ No http://en.wikipedia.org/wiki/OpenBSD[OpenBSD] C Shell, a version of http://en.wikipedia.org/wiki/C_shell[Berkeley csh] `sash` @-@popcon1@-@ @-@psize1@-@ Yes http://en.wikipedia.org/wiki/Stand-alone_shell[Stand-alone shell] with builtin commands (Not meant for standard "`/bin/sh`") `ksh` @-@popcon1@-@ @-@psize1@-@ Yes the real, AT&T version of the http://en.wikipedia.org/wiki/Korn_shell[Korn shell] `rc` @-@popcon1@-@ @-@psize1@-@ No implementation of the http://en.wikipedia.org/wiki/Plan_9_from_Bell_Labs[AT&T Plan 9] http://en.wikipedia.org/wiki/Rc[rc shell] `posh` @-@popcon1@-@ @-@psize1@-@ Yes Policy-compliant Ordinary SHell (`pdksh` derivative) ------------------------------------------------------------------------------------------------------------------------------------------ TIP: Although POSIX-like shells share the basic syntax, they can differ in behavior for things as basic as shell variables and glob expansions. Please check their documentation for details. In this tutorial chapter, the interactive shell always means `bash`. ==== Customizing bash You can customize `bash`(1) behavior by "`\~/.bashrc`". For example, try the following. -------------------- # CD upon exiting MC . /usr/lib/mc/mc.sh # set CDPATH to a good one CDPATH=.:/usr/share/doc:~:~/Desktop:~ export CDPATH PATH="${PATH}":/usr/sbin:/sbin # set PATH so it includes user's private bin if it exists if [ -d ~/bin ] ; then PATH=~/bin:"${PATH}" fi export PATH EDITOR=vim export EDITOR -------------------- TIP: You can find more `bash` customization tips, such as <<_colorized_commands>>, in <<_system_tips>>. ==== Special key strokes In the http://en.wikipedia.org/wiki/Unix-like[Unix-like] environment, there are few key strokes which have special meanings. Please note that on a normal Linux character console, only the left-hand `Ctrl` and `Alt` keys work as expected. Here are few notable key strokes to remember. .List of key bindings for bash [grid="all"] `------------------------------------------`---------------------------------------------------------------- key description of key binding ------------------------------------------------------------------------------------------------------------ `Ctrl-U` erase line before cursor `Ctrl-H` erase a character before cursor `Ctrl-D` terminate input (exit shell if you are using shell) `Ctrl-C` terminate a running program `Ctrl-Z` temporarily stop program by moving it to the background job `Ctrl-S` halt output to screen `Ctrl-Q` reactivate output to screen `Ctrl-Alt-Del` reboot/halt the system, see `inittab`(5) `Left-Alt-key` (optionally, `Windows-key`) meta-key for Emacs and the similar UI `Up-arrow` start command history search under `bash` `Ctrl-R` start incremental command history search under `bash` `Tab` complete input of the filename to the command line under `bash` `Ctrl-V` `Tab` input `Tab` without expansion to the command line under `bash` ------------------------------------------------------------------------------------------------------------ TIP: The terminal feature of `Ctrl-S` can be disabled using `stty`(1). ==== Unix style mouse operations Unix style mouse operations are based on the 3 button mouse system. .List of Unix style mouse operations [grid="all"] `-------------------------`------------------------------------------------------ action response --------------------------------------------------------------------------------- Left-click-and-drag mouse select and copy to the clipboard Left-click select the start of selection Right-click select the end of selection and copy to the clipboard Middle-click paste clipboard at the cursor --------------------------------------------------------------------------------- The center wheel on the modern wheel mouse is considered middle mouse button and can be used for middle-click. Clicking left and right mouse buttons together serves as the middle-click under the 2 button mouse system situation. In order to use a mouse in Linux character consoles, you need to have `gpm`(8) running as daemon. ==== The pager The `less`(1) command is the enhanced pager (file content browser). It reads the file specified by its command argument or its standard input. Hit "`h`" if you need help while browsing with the `less` command. It can do much more than `more`(1) and can be supercharged by executing "`eval $(lesspipe)`" or "`eval $(lessfile)`" in the shell startup script. See more in "`/usr/share/doc/lessf/LESSOPEN`". The "`-R`" option allows raw character output and enables ANSI color escape sequences. See `less`(1). ==== The text editor You should become proficient in one of variants of http://en.wikipedia.org/wiki/Vim_(text_editor)[Vim] or http://en.wikipedia.org/wiki/Emacs[Emacs] programs which are popular in the Unix-like system. I think getting used to Vim commands is the right thing to do, since Vi-editor is always there in the Linux/Unix world. (Actually, original `vi` or new `nvi` are programs you find everywhere. I chose Vim instead for newbie since it offers you help through `F1` key while it is similar enough and more powerful.) If you chose either http://en.wikipedia.org/wiki/Emacs[Emacs] or http://en.wikipedia.org/wiki/XEmacs[XEmacs] instead as your choice of the editor, that is another good choice indeed, particularly for programming. Emacs has a plethora of other features as well, including functioning as a newsreader, directory editor, mail program, etc. When used for programming or editing shell scripts, it intelligently recognizes the format of what you are working on, and tries to provide assistance. Some people maintain that the only program they need on Linux is Emacs. Ten minutes learning Emacs now can save hours later. Having the GNU Emacs manual for reference when learning Emacs is highly recommended. All these programs usually come with tutoring program for you to learn them by practice. Start Vim by typing "`vim`" and press F1-key. You should at least read the first 35 lines. Then do the online training course by moving cursor to "`|tutor|`" and pressing `Ctrl-]`. NOTE: Good editors, such as Vim and Emacs, can handle UTF-8 and other exotic encoding texts correctly. It is a good idea to use the X environment in the UTF-8 locale and to install required programs and fonts to it. Editors have options to set the file encoding independent of the X environment. Please refer to their documentation on multibyte text. ==== Setting a default text editor Debian comes with a number of different editors. We recommend to install the `vim` package, as mentioned above. Debian provides unified access to the system default editor via command "`/usr/bin/editor`" so other programs (e.g., `reportbug`(1)) can invoke it. You can change it by the following. -------------------- $ sudo update-alternatives --config editor -------------------- The choice "`/usr/bin/vim.basic`" over "`/usr/bin/vim.tiny`" is my recommendation for newbies since it supports syntax highlighting. TIP: Many programs use the environment variables "`$EDITOR`" or "`$VISUAL`" to decide which editor to use (see <<_the_internal_editor_in_mc>> and <<_customizing_program_to_be_started>>). For the consistency on the Debian system, set these to "`/usr/bin/editor`". (Historically, "`$EDITOR`" was "`ed`" and "`$VISUAL`" was "`vi`".) ==== Customizing vim You can customize `vim`(1) behavior by "`\~/.vimrc`". For example, try the following -------------------- " ------------------------------- " Local configuration " set nocompatible set nopaste set pastetoggle= syn on if $USER == "root" set nomodeline set noswapfile else set modeline set swapfile endif " filler to avoid the line above being recognized as a modeline " filler " filler -------------------- ==== Recording the shell activities The output of the shell command may roll off your screen and may be lost forever. It is a good practice to log shell activities into the file for you to review them later. This kind of record is essential when you perform any system administration tasks. The basic method of recording the shell activity is to run it under `script`(1). For example, try the following -------------------- $ script Script started, file is typescript -------------------- Do whatever shell commands under `script`. Press `Ctrl-D` to exit `script`. -------------------- $ vim typescript -------------------- See <<_recording_the_shell_activities_cleanly>> . ==== Basic Unix commands Let@@@sq@@@s learn basic Unix commands. Here I use "Unix" in its generic sense. Any Unix clone OSs usually offer equivalent commands. The Debian system is no exception. Do not worry if some commands do not work as you wish now. If `alias` is used in the shell, its corresponding command outputs are different. These examples are not meant to be executed in this order. Try all following commands from the non-privileged user account. .List of basic Unix commands [grid="all"] `--------------------------------`-------------------------------------------------------------------------------------------------------- command description ------------------------------------------------------------------------------------------------------------------------------------------ `pwd` display name of current/working directory `whoami` display current user name `id` display current user identity (name, uid, gid, and associated groups) `file ` display a type of file for the file "``" `type -p ` display a file location of command "``" `which ` , , `type ` display information on command "``" `apropos ` find commands related to "``" `man -k ` , , `whatis ` display one line explanation on command "``" `man -a ` display explanation on command "``" (Unix style) `info ` display rather long explanation on command "``" (GNU style) `ls` list contents of directory (non-dot files and directories) `ls -a` list contents of directory (all files and directories) `ls -A` list contents of directory (almost all files and directories, i.e., skip "`..`" and "`.`") `ls -la` list all contents of directory with detail information `ls -lai` list all contents of directory with inode number and detail information `ls -d` list all directories under the current directory `tree` display file tree contents `lsof ` list open status of file "``" `lsof -p ` list files opened by the process ID: "``" `mkdir ` make a new directory "``" in the current directory `rmdir ` remove a directory "``" in the current directory `cd ` change directory to the directory "``" in the current directory or in the directory listed in the variable "`$CDPATH`" `cd /` change directory to the root directory `cd` change directory to the current user@@@sq@@@s home directory `cd /` change directory to the absolute path directory "`/`" `cd ..` change directory to the parent directory `cd \~` change directory to the home directory of the user "``" `cd -` change directory to the previous directory `` create a empty file "``" `cp ` copy a existing file "``" to a new file "``" `rm ` remove a file "``" `mv ` rename an existing file "``" to a new name "``" ("``" must not exist) `mv ` move an existing file "``" to a new location "`/`" (the directory "``" must exist) `mv /` move an existing file "``" to a new location with a new name "`/`" (the directory "``" must exist but the directory "`/`" must not exist) `chmod 600 ` make an existing file "``" to be non-readable and non-writable by the other people (non-executable for all) `chmod 644 ` make an existing file "``" to be readable but non-writable by the other people (non-executable for all) `chmod 755 ` make an existing file "``" to be readable but non-writable by the other people (executable for all) `find . -name ` find matching filenames using shell "``" (slower) `locate -d . ` find matching filenames using shell "``" (quicker using regularly generated database) `grep -e "" \*.html` find a "``" in all files ending with "`.html`" in current directory and display them all `top` display process information using full screen, type "`q`" to quit `ps aux | pager` display information on all the running processes using BSD style output `ps -ef | pager` display information on all the running processes using Unix system-V style output `ps aux | grep -e "[e]xim4\*"` display all processes running "`exim`" and "`exim4`" `ps axf | pager` display information on all the running processes with ASCII art output `kill <1234>` kill a process identified by the process ID: "<1234>" `gzip ` compress "``" to create "`.gz`" using the Lempel-Ziv coding (LZ77) `gunzip .gz` decompress "`.gz`" to create "``" `bzip2 ` compress "``" to create "`.bz2`" using the Burrows-Wheeler block sorting text compression algorithm, and Huffman coding (better compression than `gzip`) `bunzip2 .bz2` decompress "`.bz2`" to create "``" `xz ` compress "``" to create "`.xz`" using the Lempel–Ziv–Markov chain algorithm (better compression than `bzip2`) `unxz .xz` decompress "`.xz`" to create "``" `tar -xvf .tar` extract files from "`.tar`" archive `tar -xvzf .tar.gz` extract files from gzipped "`.tar.gz`" archive `tar -xvjf .tar.bz2` extract files from "`.tar.bz2`" archive `tar -xvJf .tar.xz` extract files from "`.tar.xz`" archive `tar -cvf .tar /` archive contents of folder "`/`" in "`.tar`" archive `tar -cvzf .tar.gz /` archive contents of folder "`/`" in compressed "`.tar.gz`" archive `tar -cvjf .tar.bz2 /` archive contents of folder "`/`" in "`.tar.bz2`" archive `tar -cvJf .tar.xz /` archive contents of folder "`/`" in "`.tar.xz`" archive `zcat README.gz | pager` display contents of compressed "`README.gz`" using the default pager `zcat README.gz > foo` create a file "`foo`" with the decompressed content of "`README.gz`" `zcat README.gz >> foo` append the decompressed content of "`README.gz`" to the end of the file "`foo`" (if it does not exist, create it first) ------------------------------------------------------------------------------------------------------------------------------------------ NOTE: Unix has a tradition to hide filenames which start with "`.`". They are traditionally files that contain configuration information and user preferences. NOTE: For `cd` command, see `builtins`(7). NOTE: The default pager of the bare bone Debian system is `more`(1) which cannot scroll back. By installing the `less` package using command line "`apt-get install less`", `less`(1) becomes default pager and you can scroll back with cursor keys. NOTE: The "`[`" and "`]`" in the regular expression of the "`ps aux | grep -e "[e]xim4\*"`" command above enable `grep` to avoid matching itself. The "`4\*`" in the regular expression means 0 or more repeats of character "`4`" thus enables `grep` to match both "`exim`" and "`exim4`". Although "`\*`" is used in the shell filename glob and the regular expression, their meanings are different. Learn the regular expression from `grep`(1). Please traverse directories and peek into the system using the above commands as training. If you have questions on any of console commands, please make sure to read the manual page. For example, try the following -------------------- $ man man $ man bash $ man builtins $ man grep $ man ls -------------------- The style of man pages may be a little hard to get used to, because they are rather terse, particularly the older, very traditional ones. But once you get used to it, you come to appreciate their succinctness. Please note that many Unix-like commands including ones from GNU and BSD display brief help information if you invoke them in one of the following ways (or without any arguments in some cases). -------------------- $ --help $ -h -------------------- === The simple shell command Now you have some feel on how to use the Debian system. Let@@@sq@@@s look deep into the mechanism of the command execution in the Debian system. Here, I have simplified reality for the newbie. See `bash`(1) for the exact explanation. A simple command is a sequence of components. 1. Variable assignments (optional) 2. Command name 3. Arguments (optional) 4. Redirections (optional: `>` , `>>` , `<` , `<<` , etc.) 5. Control operator (optional: `&&` , `||` , , `;` , `&` , `(` , `)` ) ==== Command execution and environment variable The values of some http://en.wikipedia.org/wiki/Environment_variable[environment variables] change the behavior of some Unix commands. Default values of environment variables are initially set by the PAM system and then some of them may be reset by some application programs. - The display manager such as `gdm3` resets environment variables. - The shell in its start up codes resets environment variables in "`\~/.bash_profile`" and "`\~/.bashrc`". ==== The "`$LANG`" variable The full locale value given to "`$LANG`" variable consists of 3 parts: "`xx_YY.ZZZZ`". .The 3 parts of locale value [grid="all"] `------------`-------------------------------------------------------------------------------------------- locale value meaning ---------------------------------------------------------------------------------------------------------- `xx` http://en.wikipedia.org/wiki/ISO_639[ISO 639 language codes (lower case) such as "en"] `YY` http://en.wikipedia.org/wiki/ISO_3166-3[ISO 3166 country codes (upper case) such as "US"] `ZZZZ` http://en.wikipedia.org/wiki/Codeset[codeset, always set to "UTF-8"] ---------------------------------------------------------------------------------------------------------- For language codes and country codes, see pertinent description in the "`info gettext`". For the codeset on the modern Debian system, you should always set it to **`UTF-8`** unless you specifically want to use the historic one with good reason and background knowledge. For fine details of the locale configuration, see <<_the_locale>>. NOTE: The "`LANG=en_US`" is not "`LANG=C`" nor "`LANG=en_US.UTF-8`". It is "`LANG=en_US.ISO-8859-1`" (see <<_basics_of_encoding>>). .List of locale recommendations [grid="all"] `---------------------`------------------------- locale recommendation Language (area) ------------------------------------------------ `en_US.UTF-8` English (USA) `en_GB.UTF-8` English (Great Britain) `fr_FR.UTF-8` French (France) `de_DE.UTF-8` German (Germany) `it_IT.UTF-8` Italian (Italy) `es_ES.UTF-8` Spanish (Spain) `ca_ES.UTF-8` Catalan (Spain) `sv_SE.UTF-8` Swedish (Sweden) `pt_BR.UTF-8` Portuguese (Brazil) `ru_RU.UTF-8` Russian (Russia) `zh_CN.UTF-8` Chinese (P.R. of China) `zh_TW.UTF-8` Chinese (Taiwan R.O.C.) `ja_JP.UTF-8` Japanese (Japan) `ko_KR.UTF-8` Korean (Republic of Korea) `vi_VN.UTF-8` Vietnamese (Vietnam) ------------------------------------------------ Typical command execution uses a shell line sequence as the following. -------------------- $ date Sun Jun 3 10:27:39 JST 2007 $ LANG=fr_FR.UTF-8 date dimanche 3 juin 2007, 10:27:33 (UTC+0900) -------------------- Here, the program `date`(1) is executed with different values of the environment variable "`$LANG`". - For the first command, "`$LANG`" is set to the system default http://en.wikipedia.org/wiki/Locale[locale] value "`en_US.UTF-8`". - For the second command, "`$LANG`" is set to the French UTF-8 http://en.wikipedia.org/wiki/Locale[locale] value "`fr_FR.UTF-8`". Most command executions usually do not have preceding environment variable definition. For the above example, you can alternatively execute as the following. -------------------- $ LANG=fr_FR.UTF-8 $ date dimanche 3 juin 2007, 10:27:33 (UTC+0900) -------------------- As you can see here, the output of command is affected by the environment variable to produce French output. If you want the environment variable to be inherited to subprocesses (e.g., when calling shell script), you need to **export** it instead by the following. -------------------- $ export LANG -------------------- NOTE: When you use a typical console terminal, the "`$LANG`" environment variable is usually set to be **exported** by the desktop environment. So the above is not really a good example to test the effect of `export`. TIP: When filing a bug report, running and checking the command under "`LANG=en_US.UTF-8`" is a good idea if you use non-English environment. See `locale`(5) and `locale`(7) for "`$LANG`" and related environment variables. NOTE: I recommend you to configure the system environment just by the "`$LANG`" variable and to stay away from "`$LC_\*`" variables unless it is absolutely needed. ==== The "`$PATH`" variable When you type a command into the shell, the shell searches the command in the list of directories contained in the "`$PATH`" environment variable. The value of the "`$PATH`" environment variable is also called the shell@@@sq@@@s search path. In the default Debian installation, the "`$PATH`" environment variable of user accounts may not include "`/sbin`" and "`/usr/sbin`". For example, the `ifconfig` command needs to be issued with full path as "`/sbin/ifconfig`". (Similar `ip` command is located in "`/bin`".) You can change the "`$PATH`" environment variable of Bash shell by "`\~/.bash_profile`" or "`\~/.bashrc`" files. ==== The "`$HOME`" variable Many commands stores user specific configuration in the home directory and changes their behavior by their contents. The home directory is identified by the environment variable "`$HOME`". .List of "`$HOME`" values [grid="all"] `---------------------`------------------------------------------------- value of "`$HOME`" program execution situation ------------------------------------------------------------------------ `/` program run by the init process (daemon) `/root` program run from the normal root shell `/home/` program run from the normal user shell `/home/` program run from the normal user GUI desktop menu `/home/` program run as root with "`sudo program`" `/root` program run as root with "`sudo -H program`" ------------------------------------------------------------------------ TIP: Shell expands "`\~/`" to current user's home directory, i.e., "`$HOME/`". Shell expands "`\~foo/`" to `foo`@@@sq@@@s home directory, i.e., "`/home/foo/`". ==== Command line options Some commands take arguments. Arguments starting with "`-`" or "`--`" are called options and control the behavior of the command. -------------------- $ date Mon Oct 27 23:02:09 CET 2003 $ date -R Mon, 27 Oct 2003 23:02:40 +0100 -------------------- Here the command-line argument "`-R`" changes `date`(1) behavior to output http://tools.ietf.org/html/rfc2822[RFC2822] compliant date string. ==== Shell glob Often you want a command to work with a group of files without typing all of them. The filename expansion pattern using the shell **glob**, (sometimes referred as **wildcards**), facilitate this need. .Shell glob patterns [grid="all"] `------------------`---------------------------------------------------------------------------------------------------- shell glob pattern description of match rule ------------------------------------------------------------------------------------------------------------------------ `\*` filename (segment) not started with "`.`" `.\*` filename (segment) started with "`.`" `?` exactly one character `[...]` exactly one character with any character enclosed in brackets `[a-z]` exactly one character with any character between "`a`" and "`z`" `[\^...]` exactly one character other than any character enclosed in brackets (excluding "`\^`") ------------------------------------------------------------------------------------------------------------------------ For example, try the following -------------------- $ mkdir junk; cd junk; touch 1.txt 2.txt 3.c 4.h .5.txt ..6.txt $ echo *.txt 1.txt 2.txt $ echo * 1.txt 2.txt 3.c 4.h $ echo *.[hc] 3.c 4.h $ echo .* . .. .5.txt ..6.txt $ echo .*[^.]* .5.txt ..6.txt $ echo [^1-3]* 4.h $ cd ..; rm -rf junk -------------------- See `glob`(7). NOTE: Unlike normal filename expansion by the shell, the shell pattern "`\*`" tested in `find`(1) with "`-name`" test etc., matches the initial "`.`" of the filename. (New http://en.wikipedia.org/wiki/POSIX[POSIX] feature) NOTE: BASH can be tweaked to change its glob behavior with its shopt builtin options such as "`dotglob`", "`noglob`", "`nocaseglob`", "`nullglob`", "`extglob`", etc. See `bash`(1). ==== Return value of the command Each command returns its exit status (variable: "`$?`") as the return value. .Command exit codes [grid="all"] `-------------------`--------------------`-------------------- command exit status numeric return value logical return value -------------------------------------------------------------- success zero, 0 **TRUE** error non-zero, -1 **FALSE** -------------------------------------------------------------- For example, try the following. -------------------- $ [ 1 = 1 ] ; echo $? 0 $ [ 1 = 2 ] ; echo $? 1 -------------------- NOTE: Please note that, in the logical context for the shell, **success** is treated as the logical **TRUE** which has 0 (zero) as its value. This is somewhat non-intuitive and needs to be reminded here. ==== Typical command sequences and shell redirection Let@@@sq@@@s try to remember following shell command idioms typed in one line as a part of shell command. .Shell command idioms [grid="all"] `--------------------------`-------------------------------------------------------------------------------------------------------------- command idiom description ------------------------------------------------------------------------------------------------------------------------------------------ `command &` **background** execution of `command` in the subshell `command1 | command2` **pipe** the standard output of `command1` to the standard input of `command2` (**concurrent** execution) `command1 2>&1 | command2` **pipe** both standard output and standard error of `command1` to the standard input of `command2` (**concurrent** execution) `command1 ; command2` execute `command1` and `command2` **sequentially** `command1 && command2` execute `command1`; if successful, execute `command2` **sequentially** (return success if both `command1` **and** `command2` are successful) `command1 || command2` execute `command1`; if not successful, execute `command2` **sequentially** (return success if `command1` **or** `command2` are successful) `command > foo` redirect standard output of `command` to a file `foo` (overwrite) `command 2> foo` redirect standard error of `command` to a file `foo` (overwrite) `command >> foo` redirect standard output of `command` to a file `foo` (append) `command 2>> foo` redirect standard error of `command` to a file `foo` (append) `command > foo 2>&1` redirect both standard output and standard error of `command` to a file `foo` `command < foo` redirect standard input of `command` to a file `foo` `command << delimiter` redirect standard input of `command` to the following lines until "`delimiter`" is met (here document) `command <\<- delimiter` redirect standard input of `command` to the following lines until "`delimiter`" is met (here document, the leading tab characters are stripped from input lines) ------------------------------------------------------------------------------------------------------------------------------------------ The Debian system is a multi-tasking system. Background jobs allow users to run multiple programs in a single shell. The management of the background process involves the shell builtins: `jobs`, `fg`, `bg`, and `kill`. Please read sections of bash(1) under "SIGNALS", and "JOB CONTROL", and `builtins`(1). For example, try the following -------------------- $ foo $ exec 3bar # open files $ cat <&3 >&4 # redirect stdin to 3, stdout to 4 $ exec 3<&- 4>&- # close files $ cat bar Hello -------------------- The file descriptor 0-2 are predefined. .Predefined file descriptors [grid="all"] `--------`---------------`--------------- device description file descriptor ----------------------------------------- `stdin` standard input 0 `stdout` standard output 1 `stderr` standard error 2 ----------------------------------------- ==== Command alias You can set an alias for the frequently used command. For example, try the following -------------------- $ alias la='ls -la' -------------------- Now, "`la`" works as a short hand for "`ls -la`" which lists all files in the long listing format. You can list any existing aliases by `alias` (see `bash`(1) under "SHELL BUILTIN COMMANDS"). -------------------- $ alias ... alias la='ls -la' -------------------- You can identity exact path or identity of the command by `type` (see `bash`(1) under "SHELL BUILTIN COMMANDS"). For example, try the following -------------------- $ type ls ls is hashed (/bin/ls) $ type la la is aliased to ls -la $ type echo echo is a shell builtin $ type file file is /usr/bin/file -------------------- Here `ls` was recently searched while "`file`" was not, thus "`ls`" is "hashed", i.e., the shell has an internal record for the quick access to the location of the "`ls`" command. TIP: See <<_colorized_commands>>. === Unix-like text processing In Unix-like work environment, text processing is done by piping text through chains of standard text processing tools. This was another crucial Unix innovation. ==== Unix text tools There are few standard text processing tools which are used very often on the Unix-like system. - No regular expression is used: * `cat`(1) concatenates files and outputs the whole content. * `tac`(1) concatenates files and outputs in reverse. * `cut`(1) selects parts of lines and outputs. * `head`(1) outputs the first part of files. * `tail`(1) outputs the last part of files. * `sort`(1) sorts lines of text files. * `uniq`(1) removes duplicate lines from a sorted file. * `tr`(1) translates or deletes characters. * `diff`(1) compares files line by line. - Basic regular expression (**BRE**) is used: * `grep`(1) matches text with patterns. * `ed`(1) is a primitive line editor. * `sed`(1) is a stream editor. * `vim`(1) is a screen editor. * `emacs`(1) is a screen editor. (somewhat extended **BRE**) - Extended regular expression (**ERE**) is used: * `egrep`(1) matches text with patterns. * `awk`(1) does simple text processing. * `tcl`(3tcl) can do every conceivable text processing: See `re_syntax`(3). Often used with `tk`(3tk). * `perl`(1) can do every conceivable text processing. See `perlre`(1). * `pcregrep`(1) from the `pcregrep` package matches text with http://en.wikipedia.org/wiki/Perl_Compatible_Regular_Expressions[Perl Compatible Regular Expressions (PCRE)] pattern. * `python`(1) with the `re` module can do every conceivable text processing. See "`/usr/share/doc/python/html/index.html`". If you are not sure what exactly these commands do, please use "`man command`" to figure it out by yourself. NOTE: Sort order and range expression are locale dependent. If you wish to obtain traditional behavior for a command, use **C** locale instead of **UTF-8** ones by prepending command with "`LANG=C`" (see <<_the_literal_lang_literal_variable>> and <<_the_locale>>). NOTE: http://en.wikipedia.org/wiki/Perl[Perl] regular expressions (`perlre`(1)), http://en.wikipedia.org/wiki/Perl_Compatible_Regular_Expressions[Perl Compatible Regular Expressions (PCRE)], and http://en.wikipedia.org/wiki/Python_(programming_language)[Python] regular expressions offered by the `re` module have many common extensions to the normal **ERE**. ==== Regular expressions http://en.wikipedia.org/wiki/Regular_expression[Regular expressions] are used in many text processing tools. They are analogous to the shell globs, but they are more complicated and powerful. The regular expression describes the matching pattern and is made up of text characters and **metacharacters**. A **metacharacter** is just a character with a special meaning. There are 2 major styles, **BRE** and **ERE**, depending on the text tools as described above. .Metacharacters for BRE and ERE [grid="all"] `------------------------`--------------------`------------------------------------------------------------------------------------------- BRE ERE description of the regular expression ------------------------------------------------------------------------------------------------------------------------------------------ `\ . [ ] \^ $ \*` `\ . [ ] \^ $ \*` common **metacharacters** `\\+ \? \( \) \\{ \\} \|` {nbsp} BRE only "`\`" escaped **metacharacters** {nbsp} `\+ ? ( ) { } |` ERE only non-"`\`" escaped **metacharacters** `c` `c` match **non-metacharacter** "`c`" `\c` `\c` match a literal character "`c`" even if "`c`" is **metacharacter** by itself `.` `.` match any character including newline `\^` `\^` position at the beginning of a string `$` `$` position at the end of a string `\<` `\<` position at the beginning of a word `\>` `\>` position at the end of a word `[abc...]` `[abc...]` match any characters in "`abc...`" `[\^abc...]` `[\^abc...]` match any characters except in "`abc...`" `r\*` `r\*` match zero or more regular expressions identified by "`r`" `r\\+` `r\+` match one or more regular expressions identified by "`r`" `r\?` `r?` match zero or one regular expressions identified by "`r`" `r1\|r2` `r1|r2` match one of the regular expressions identified by "`r1`" or "`r2`" `\(r1\|r2\)` `(r1|r2)` match one of the regular expressions identified by "`r1`" or "`r2`" and treat it as a **bracketed** regular expression ------------------------------------------------------------------------------------------------------------------------------------------ The regular expression of **`emacs`** is basically **BRE** but has been extended to treat "`\+`"and "`?`" as the **metacharacters** as in **ERE**. Thus, there are no needs to escape them with "`\`" in the regular expression of `emacs`. `grep`(1) can be used to perform the text search using the regular expression. For example, try the following -------------------- $ egrep 'GNU.*LICENSE|Yoyodyne' /usr/share/common-licenses/GPL GNU GENERAL PUBLIC LICENSE GNU GENERAL PUBLIC LICENSE Yoyodyne, Inc., hereby disclaims all copyright interest in the program -------------------- TIP: See <<_colorized_commands>>. ==== Replacement expressions For the replacement expression, some characters have special meanings. .The replacement expression [grid="all"] `----------------------`----------------------------------------------------------------------------- replacement expression description of the text to replace the replacement expression ----------------------------------------------------------------------------------------------------- `&` what the regular expression matched (use `\&` in `emacs`) `\n` what the n-th **bracketed** regular expression matched ("n" being number) ----------------------------------------------------------------------------------------------------- For Perl replacement string, "`$n`" is used instead of "`\n`" and "`&`" has no special meaning. For example, try the following -------------------- $ echo zzz1abc2efg3hij4 | \ sed -e 's/\(1[a-z]*\)[0-9]*\(.*\)$/=&=/' zzz=1abc2efg3hij4= $ echo zzz1abc2efg3hij4 | \ sed -e 's/\(1[a-z]*\)[0-9]*\(.*\)$/\2===\1/' zzzefg3hij4===1abc $ echo zzz1abc2efg3hij4 | \ perl -pe 's/(1[a-z]*)[0-9]*(.*)$/$2===$1/' zzzefg3hij4===1abc $ echo zzz1abc2efg3hij4 | \ perl -pe 's/(1[a-z]*)[0-9]*(.*)$/=&=/' zzz=&= -------------------- Here please pay extra attention to the style of the **bracketed** regular expression and how the matched strings are used in the text replacement process on different tools. These regular expressions can be used for cursor movements and text replacement actions in some editors too. The back slash "`\`" at the end of line in the shell commandline escapes newline as a white space character and continues shell command line input to the next line. Please read all the related manual pages to learn these commands. ==== Global substitution with regular expressions The `ed`(1) command can replace all instances of "`FROM_REGEX`" with "`TO_TEXT`" in "`file`". -------------------- $ ed file <, , and combined. Be careful about using this shell IFS tricks. Strange things may happen, when shell interprets some parts of the script as its **input**. -------------------- $ IFS=":," # use ":" and "," as IFS $ echo IFS=$IFS, IFS="$IFS" # echo is a Bash builtin IFS= , IFS=:, $ date -R # just a command output Sat, 23 Aug 2003 08:30:15 +0200 $ echo $(date -R) # sub shell --> input to main shell Sat 23 Aug 2003 08 30 36 +0200 $ unset IFS # reset IFS to the default $ echo $(date -R) Sat, 23 Aug 2003 08:30:50 +0200 -------------------- ==== Script snippets for piping commands The following scripts do nice things as a part of a pipe. .List of script snippets for piping commands [grid="all"] `----------------------------------`------------------------------------------------------------------------- script snippet (type in one line) effect of command ------------------------------------------------------------------------------------------------------------- `find /usr -print` find all files under "`/usr`" `seq 1 100` print 1 to 100 `| xargs -n 1 ` run command repeatedly with each item from pipe as its argument `| xargs -n 1 echo` split white-space-separated items from pipe into lines `| xargs echo` merge all lines from pipe into a line `| grep -e ` extract lines from pipe containing `| grep -v -e ` extract lines from pipe not containing `| cut -d: -f3 -` extract third field from pipe separated by "`:`" (passwd file etc.) `| awk '{ print $3 }'` extract third field from pipe separated by whitespaces `| awk -F'\t' '{ print $3 }'` extract third field from pipe separated by tab `| col -bx` remove backspace and expand tabs to spaces `| expand -` expand tabs `| sort| uniq` sort and remove duplicates `| tr 'A-Z' 'a-z'` convert uppercase to lowercase `| tr -d '\n'` concatenate lines into one line `| tr -d '\r'` remove CR `| sed 's/\^/# /'` add "`#`" to the start of each line `| sed 's/\.ext//g'` remove "`.ext`" `| sed -n -e 2p` print the second line `| head -n 2 -` print the first 2 lines `| tail -n 2 -` print the last 2 lines ------------------------------------------------------------------------------------------------------------- A one-line shell script can loop over many files using `find`(1) and `xargs`(1) to perform quite complicated tasks. See <<_idioms_for_the_selection_of_files>> and <<_repeating_a_command_looping_over_files>>. When using the shell interactive mode becomes too complicated, please consider to write a shell script (see <<_the_shell_script>>). debian-reference-2.53/asciidoc/12_program.txt0000644000000000000000000011503412255345173016012 0ustar == Programming // vim: set sts=2 expandtab: // Use ":set nowrap" to edit table I provide some pointers for people to learn programming on the Debian system enough to trace the packaged source code. Here are notable packages and corresponding documentation packages for programing. .List of packages to help programing [grid="all"] `-----------------`-------------`------------`------------------------------------------------------------------------ package popcon size documentation ---------------------------------------------------------------------------------------------------------------------- `autoconf` @-@popcon1@-@ @-@psize1@-@ "`info autoconf`" provided by `autoconf-doc` `automake` @-@popcon1@-@ @-@psize1@-@ "`info automake`" provided by `automake1.10-doc` `bash` @-@popcon1@-@ @-@psize1@-@ "`info bash`" provided by `bash-doc` `bison` @-@popcon1@-@ @-@psize1@-@ "`info bison`" provided by `bison-doc` `cpp` @-@popcon1@-@ @-@psize1@-@ "`info cpp`" provided by `cpp-doc` `ddd` @-@popcon1@-@ @-@psize1@-@ "`info ddd`" provided by `ddd-doc` `exuberant-ctags` @-@popcon1@-@ @-@psize1@-@ `exuberant-ctags`(1) `flex` @-@popcon1@-@ @-@psize1@-@ "`info flex`" provided by `flex-doc` `gawk` @-@popcon1@-@ @-@psize1@-@ "`info gawk`" provided by `gawk-doc` `gcc` @-@popcon1@-@ @-@psize1@-@ "`info gcc`" provided by `gcc-doc` `gdb` @-@popcon1@-@ @-@psize1@-@ "`info gdb`" provided by `gdb-doc` `gettext` @-@popcon1@-@ @-@psize1@-@ "`info gettext`" provided by `gettext-doc` `gfortran` @-@popcon1@-@ @-@psize1@-@ "`info gfortran`" provided by `gfortran-doc` (Fortran 95) `fpc` @-@popcon1@-@ @-@psize1@-@ `fpc`(1) and html by `fp-docs` (Pascal) `glade` @-@popcon1@-@ @-@psize1@-@ help provided via menu (UI Builder) `libc6` @-@popcon1@-@ @-@psize1@-@ "`info libc`" provided by `glibc-doc` and `glibc-doc-reference` `make` @-@popcon1@-@ @-@psize1@-@ "`info make`" provided by `make-doc` `xutils-dev` @-@popcon1@-@ @-@psize1@-@ `imake`(1), `xmkmf`(1), etc. `mawk` @-@popcon1@-@ @-@psize1@-@ `mawk`(1) `perl` @-@popcon1@-@ @-@psize1@-@ `perl`(1) and html pages provided by `perl-doc` and `perl-doc-html` `python` @-@popcon1@-@ @-@psize1@-@ `python`(1) and html pages provided by `python-doc` `tcl8.4` @-@popcon1@-@ @-@psize1@-@ `tcl`(3) and detail manual pages provided by `tcl8.4-doc` `tk8.4` @-@popcon1@-@ @-@psize1@-@ `tk`(3) and detail manual pages provided by `tk8.4-doc` `ruby` @-@popcon1@-@ @-@psize1@-@ `ruby`(1) and interactive reference provided by `ri` `vim` @-@popcon1@-@ @-@psize1@-@ help(F1) menu provided by `vim-doc` `susv2` @-@popcon1@-@ @-@psize1@-@ fetch "http://www.unix.org/version2/[The Single Unix Specifications v2]" `susv3` @-@popcon1@-@ @-@psize1@-@ fetch "http://www.unix.org/version3/[The Single Unix Specifications v3]" ---------------------------------------------------------------------------------------------------------------------- Online references are available by typing "`man name`" after installing `manpages` and `manpages-dev` packages. Online references for the GNU tools are available by typing "`info program_name`" after installing the pertinent documentation packages. You may need to include the `contrib` and `non-free` archives in addition to the `main` archive since some GFDL documentations are not considered to be DFSG compliant. WARNING: Do not use "`test`" as the name of an executable test file. "`test`" is a shell builtin. CAUTION: You should install software programs directly compiled from source into "`/usr/local`" or "`/opt`" to avoid collision with system programs. TIP: http://www.99-bottles-of-beer.net/[Code examples of creating "Song 99 Bottles of Beer"] should give you good ideas of practically all the programming languages. === The shell script The http://en.wikipedia.org/wiki/Shell_script[shell script] is a text file with the execution bit set and contains the commands in the following format. -------------------- #!/bin/sh ... command lines -------------------- The first line specifies the shell interpreter which read and execute this file contents. Reading shell scripts is the **best** way to understand how a Unix-like system works. Here, I give some pointers and reminders for shell programming. See "Shell Mistakes" (http://www.greenend.org.uk/rjk/2001/04/shell.html) to learn from mistakes. Unlike shell interactive mode (see <<_the_simple_shell_command>> and <<_unix_like_text_processing>>), shell scripts frequently use parameters, conditionals, and loops. ==== POSIX shell compatibility Many system scripts may be interpreted by any one of http://en.wikipedia.org/wiki/POSIX[POSIX] shells (see <>). The default shell for the system is "`/bin/sh`" which is a symlink pointing to the actual program. - `bash`(1) for `lenny` or older - `dash`(1) for `squeeze` or newer Avoid writing a shell script with **bashisms** or **zshisms** to make it portable among all POSIX shells. You can check it using `checkbashisms`(1). .List of typical bashisms [grid="all"] `-----------------------------------`------------------------------------ Good: POSIX Avoid: bashism ------------------------------------------------------------------------- `if [ "$foo" = "$bar" ] ; then ...` `if [ "$foo" == "$bar" ] ; then ...` `diff -u file.c.orig file.c` `diff -u file.c{.orig,}` `mkdir /foobar /foobaz` `mkdir /foo{bar,baz}` `funcname() { ... }` `function funcname() { ... }` octal format: "`\377`" hexadecimal format: "`\xff`" ------------------------------------------------------------------------- The "`echo`" command must be used with following cares since its implementation differs among shell builtin and external commands. - Avoid using command option "`-e`" and "`-E`". - Avoid using any command options except "`-n`". - Avoid using escape sequences in the string since their handling varies. NOTE: Although "`-n`" option is **not** really POSIX syntax, it is generally accepted. TIP: Use the "`printf`" command instead of the "`echo`" command if you need to embed escape sequences in the output string. ==== Shell parameters Special shell parameters are frequently used in the shell script. .List of shell parameters [grid="all"] `---------------`------------------------------------------- shell parameter value ------------------------------------------------------------ `$0` name of the shell or shell script `$1` first(1) shell argument `$9` ninth(9) shell argument `$#` number of positional parameters `"$\*"` `"$1 $2 $3 $4 ... "` `"$@"` `"$1" "$2" "$3" "$4" ...` `$?` exit status of the most recent command `$$` PID of this shell script `$!` PID of most recently started background job ------------------------------------------------------------ Basic **parameter expansions** to remember are followings. .List of shell parameter expansions [grid="all"] `-------------------------`---------------------`--------------------------------------------------- parameter expression form value if `var` is set value if `var` is not set ---------------------------------------------------------------------------------------------------- `$\{var:-string\}` "`$var`" "`string`" `$\{var:+string\}` "`string`" "`null`" `$\{var:=string\}` "`$var`" "`string`" (and run "`var=string`") `$\{var:?string\}` "`$var`" echo "`string`" to **stderr** (and exit with error) ---------------------------------------------------------------------------------------------------- Here, the colon "`:`" in all of these operators is actually optional. - **with** "`:`" = operator test for **exist** and **not null** - **without** "`:`" = operator test for **exist** only .List of key shell parameter substitutions [grid="all"] `---------------------------`------------------------------ parameter substitution form result ----------------------------------------------------------- `$\{var%suffix\}` remove smallest suffix pattern `$\{var%%suffix\}` remove largest suffix pattern `$\{var#prefix\}` remove smallest prefix pattern `$\{var##prefix\}` remove largest prefix pattern ----------------------------------------------------------- ==== Shell conditionals Each command returns an **exit status** which can be used for conditional expressions. - Success: 0 ("True") - Error: non 0 ("False") NOTE: "0" in the shell conditional context means "True", while "0" in the C conditional context means "False". NOTE: "`[`" is the equivalent of the `test` command, which evaluates its arguments up to "`]`" as a conditional expression. Basic **conditional idioms** to remember are followings. - "` && || true`" - "` || || true`" - A multi-line script snippet as the following -------------------- if [ ]; then else fi -------------------- Here trailing "`|| true`" was needed to ensure this shell script does not exit at this line accidentally when shell is invoked with "`-e`" flag. .List of file comparison operators in the conditional expression [grid="all"] `-----------------------`------------------------------------------------------ equation condition to return logical true ------------------------------------------------------------------------------- `-e ` exists `-d ` exists and is a directory `-f ` exists and is a regular file `-w ` exists and is writable `-x ` exists and is executable ` -nt ` is newer than (modification) ` -ot ` is older than (modification) ` -ef ` and are on the same device and the same inode number ------------------------------------------------------------------------------- .List of string comparison operators in the conditional expression [grid="all"] `--------------------`------------------------------------------------------ equation condition to return logical true ---------------------------------------------------------------------------- `-z ` the length of is zero `-n ` the length of is non-zero ` = ` and are equal ` != ` and are not equal ` < ` sorts before (locale dependent) ` > ` sorts after (locale dependent) ---------------------------------------------------------------------------- **Arithmetic** integer comparison operators in the conditional expression are "`-eq`", "`-ne`", "`-lt`", "`-le`", "`-gt`", and "`-ge`". ==== Shell loops There are several loop idioms to use in POSIX shell. - "`for x in foo1 foo2 ... ; do command ; done`" loops by assigning items from the list "`foo1 foo2 ...`" to variable "`x`" and executing "`command`". - "`while condition ; do command ; done`" repeats "`command`" while "`condition`" is true. - "`until condition ; do command ; done`" repeats "`command`" while "`condition`" is not true. - "`break`" enables to exit from the loop. - "`continue`" enables to resume the next iteration of the loop. TIP: The http://en.wikipedia.org/wiki/C_(programming_language)[C]-language like numeric iteration can be realized by using `seq`(1) as the "`foo1 foo2 ...`" generator. TIP: See <<_repeating_a_command_looping_over_files>>. ==== The shell command-line processing sequence The shell processes a script roughly as the following sequence. // following bash manpage (but reserved word are picked from common ones with dash - The shell reads a line. - The shell groups a part of the line as **one token** if it is within `"…"` or `'…'`. - The shell splits other part of a line into **tokens** by the following. * Whitespaces: ` ` * Metacharacters: `< > | ; & ( )` - The shell checks the **reserved word** for each token to adjust its behavior if not within `"..."` or `'...'`. * **reserved word**: `if then elif else fi for in while unless do done case esac` - The shell expands **alias** if not within `"..."` or `'...'`. - The shell expands **tilde** if not within `"..."` or `'...'`. * "`\~`" -> current user@@@sq@@@s home directory * "`\~`" -> ``@@@sq@@@s home directory - The shell expands **parameter** to its value if not within `'...'`. * **parameter**: "`$PARAMETER`" or "`$\{PARAMETER\}`" - The shell expands **command substitution** if not within `'...'`. * "`$( command )`" -> the output of "`command`" * "`@@@grave@@@ command @@@grave@@@`" -> the output of "`command`" - The shell expands **pathname glob** to matching file names if not within `"..."` or `'...'`. * `\*` -> any characters * `?` -> one character * `[...]` -> any one of the characters in "`...`" - The shell looks up **command** from the following and execute it. * **function** definition * **builtin** command * **executable file** in "`$PATH`" - The shell goes to the next line and repeats this process again from the top of this sequence. Single quotes within double quotes have no effect. Executing "`set -x`" in the shell or invoking the shell with "`-x`" option make the shell to print all of commands executed. This is quite handy for debugging. ==== Utility programs for shell script In order to make your shell program as portable as possible across Debian systems, it is a good idea to limit utility programs to ones provided by **essential** packages. - "`aptitude search \~E`" lists **essential** packages. - "`dpkg -L |grep '/man/man.\*/'`" lists manpages for commands offered by `` package. .List of packages containing small utility programs for shell scripts [grid="all"] `--------------`-------------`------------`---------------------------------------------- package popcon size description ----------------------------------------------------------------------------------------- `coreutils` @-@popcon1@-@ @-@psize1@-@ GNU core utilities `debianutils` @-@popcon1@-@ @-@psize1@-@ miscellaneous utilities specific to Debian `bsdmainutils` @-@popcon1@-@ @-@psize1@-@ collection of more utilities from FreeBSD `bsdutils` @-@popcon1@-@ @-@psize1@-@ basic utilities from 4.4BSD-Lite `moreutils` @-@popcon1@-@ @-@psize1@-@ additional Unix utilities ------------------------------------------------------------------------------------------ TIP: Although `moreutils` may not exist ouside of Debian, it offers interesting small programs. Most notable one is `sponge`(8) which is quite useful when you wish to overwrite original file. ==== Shell script dialog The user interface of a simple shell program can be improved from dull interaction by `echo` and `read` commands to more interactive one by using one of the so-called dialog program etc. .List of user interface programs [grid="all"] `-----------`-------------`------------`----------------------------------------------------------------------------------- package popcon size description --------------------------------------------------------------------------------------------------------------------------- `x11-utils` @-@popcon1@-@ @-@psize1@-@ `xmessage`(1): display a message or query in a window (X) `whiptail` @-@popcon1@-@ @-@psize1@-@ displays user-friendly dialog boxes from shell scripts (newt) `dialog` @-@popcon1@-@ @-@psize1@-@ displays user-friendly dialog boxes from shell scripts (ncurses) `zenity` @-@popcon1@-@ @-@psize1@-@ display graphical dialog boxes from shell scripts (gtk2.0) `ssft` @-@popcon1@-@ @-@psize1@-@ Shell Scripts Frontend Tool (wrapper for zenity, kdialog, and dialog with gettext) `gettext` @-@popcon1@-@ @-@psize1@-@ "`/usr/bin/gettext.sh`": translate message --------------------------------------------------------------------------------------------------------------------------- ==== Shell script example with zenity Here is a simple script which creates ISO image with RS02 data supplemented by `dvdisaster`(1). -------------------- #!/bin/sh -e # gmkrs02 : Copyright (C) 2007 Osamu Aoki , Public Domain #set -x error_exit() { echo "$1" >&2 exit 1 } # Initialize variables DATA_ISO="$HOME/Desktop/iso-$$.img" LABEL=$(date +%Y%m%d-%H%M%S-%Z) if [ $# != 0 ] && [ -d "$1" ]; then DATA_SRC="$1" else # Select directory for creating ISO image from folder on desktop DATA_SRC=$(zenity --file-selection --directory \ --title="Select the directory tree root to create ISO image") \ || error_exit "Exit on directory selection" fi # Check size of archive xterm -T "Check size $DATA_SRC" -e du -s $DATA_SRC/* SIZE=$(($(du -s $DATA_SRC | awk '{print $1}')/1024)) if [ $SIZE -le 520 ] ; then zenity --info --title="Dvdisaster RS02" --width 640 --height 400 \ --text="The data size is good for CD backup:\\n $SIZE MB" elif [ $SIZE -le 3500 ]; then zenity --info --title="Dvdisaster RS02" --width 640 --height 400 \ --text="The data size is good for DVD backup :\\n $SIZE MB" else zenity --info --title="Dvdisaster RS02" --width 640 --height 400 \ --text="The data size is too big to backup : $SIZE MB" error_exit "The data size is too big to backup :\\n $SIZE MB" fi # only xterm is sure to have working -e option # Create raw ISO image rm -f "$DATA_ISO" || true xterm -T "genisoimage $DATA_ISO" \ -e genisoimage -r -J -V "$LABEL" -o "$DATA_ISO" "$DATA_SRC" # Create RS02 supplemental redundancy xterm -T "dvdisaster $DATA_ISO" -e dvdisaster -i "$DATA_ISO" -mRS02 -c zenity --info --title="Dvdisaster RS02" --width 640 --height 400 \ --text="ISO/RS02 data ($SIZE MB) \\n created at: $DATA_ISO" # EOF -------------------- You may wish to create launcher on the desktop with command set something like "`/usr/local/bin/gmkrs02 %d`". === Make http://en.wikipedia.org/wiki/Make_(software)[Make] is a utility to maintain groups of programs. Upon execution of `make`(1), `make` read the rule file, "`Makefile`", and updates a target if it depends on prerequisite files that have been modified since the target was last modified, or if the target does not exist. The execution of these updates may occur concurrently. The rule file syntax is the following. -------------------- target: [ prerequisites ... ] [TAB] command1 [TAB] -command2 # ignore errors [TAB] @command3 # suppress echoing -------------------- Here "`[TAB]`" is a TAB code. Each line is interpreted by the shell after make variable substitution. Use "`\`" at the end of a line to continue the script. Use "`$$`" to enter "`$`" for environment values for a shell script. Implicit rules for the target and prerequisites can be written, for example, by the following. -------------------- %.o: %.c header.h -------------------- Here, the target contains the character "`%`" (exactly one of them). The "`%`" can match any nonempty substring in the actual target filenames. The prerequisites likewise use "`%`" to show how their names relate to the actual target name. .List of make automatic variables [grid="all"] `------------------`---------------------------------------- automatic variable value ------------------------------------------------------------ `$@` target `$<` first prerequisite `$?` all newer prerequisites `$\^` all prerequisites `$\*` "`%`" matched stem in the target pattern ------------------------------------------------------------ .List of make variable expansions [grid="all"] `------------------`------------------- variable expansion description --------------------------------------- `foo1 := bar` one-time expansion `foo2 = bar` recursive expansion `foo3 += bar` append --------------------------------------- Run "`make -p -f/dev/null`" to see automatic internal rules. === C You can set up proper environment to compile programs written in the http://en.wikipedia.org/wiki/C_(programming_language)[C programming language] by the following. -------------------- # apt-get install glibc-doc manpages-dev libc6-dev gcc build-essential -------------------- The `libc6-dev` package, i.e., GNU C Library, provides http://en.wikipedia.org/wiki/C_standard_library[C standard library] which is collection of header files and library routines used by the C programming language. See references for C as the following. - "`info libc`" (C library function reference) - `gcc`(1) and "`info gcc`" - `each_C_library_function_name`(3) - Kernighan & Ritchie, "The C Programming Language", 2nd edition (Prentice Hall) ==== Simple C program (gcc) A simple example "`example.c`" can compiled with a library "`libm`" into an executable "`run_example`" by the following. -------------------- $ cat > example.c << EOF #include #include #include int main(int argc, char **argv, char **envp){ double x; char y[11]; x=sqrt(argc+7.5); strncpy(y, argv[0], 10); /* prevent buffer overflow */ y[10] = '\0'; /* fill to make sure string ends with '\0' */ printf("%5i, %5.3f, %10s, %10s\n", argc, x, y, argv[1]); return 0; } EOF $ gcc -Wall -g -o run_example example.c -lm $ ./run_example 1, 2.915, ./run_exam, (null) $ ./run_example 1234567890qwerty 2, 3.082, ./run_exam, 1234567890qwerty -------------------- Here, "`-lm`" is needed to link library "`/usr/lib/libm.so`" from the `libc6` package for `sqrt`(3). The actual library is in "`/lib/`" with filename "`libm.so.6`", which is a symlink to "`libm-2.7.so`". Look at the last parameter in the output text. There are more than 10 characters even though "`%10s`" is specified. The use of pointer memory operation functions without boundary checks, such as `sprintf`(3) and `strcpy`(3), is deprecated to prevent buffer overflow exploits that leverage the above overrun effects. Instead, use `snprintf`(3) and `strncpy`(3). === Debug Debug is important part of programing activities. Knowing how to debug programs makes you a good Debian user who can produce meaningful bug reports. ==== Basic gdb execution Primary http://en.wikipedia.org/wiki/Debugger[debugger] on Debian is `gdb`(1) which enables you to inspect a program while it executes. Let's install `gdb` and related programs by the following. -------------------- # apt-get install gdb gdb-doc build-essential devscripts -------------------- Good tutorial of `gdb` is provided by "`info gdb`" or found http://www.unknownroad.com/rtfm/gdbtut/gdbtoc.html[elsewhere on the web]. Here is a simple example of using `gdb`(1) on a "`program`" compiled with the "`-g`" option to produce debugging information. -------------------- $ gdb program (gdb) b 1 # set break point at line 1 (gdb) run args # run program with args (gdb) next # next line ... (gdb) step # step forward ... (gdb) p parm # print parm ... (gdb) p parm=12 # set value to 12 ... (gdb) quit -------------------- TIP: Many `gdb`(1) commands can be abbreviated. Tab expansion works as in the shell. ==== Debugging the Debian package Since all installed binaries should be stripped on the Debian system by default, most debugging symbols are removed in the normal package. In order to debug Debian packages with `gdb`(1), corresponding `\*-dbg` packages need to be installed (e.g. `libc6-dbg` in the case of `libc6`). If a package to be debugged does not provide its `\*-dbg` package, you need to install it after rebuilding it by the following. -------------------- $ mkdir /path/new ; cd /path/new $ sudo apt-get update $ sudo apt-get dist-upgrade $ sudo apt-get install fakeroot devscripts build-essential $ sudo apt-get build-dep source_package_name $ apt-get source package_name $ cd package_name* -------------------- Fix bugs if needed. Bump package version to one which does not collide with official Debian versions, e.g. one appended with "`+debug1`" when recompiling existing package version, or one appended with "`\~pre1`" when compiling unreleased package version by the following. -------------------- $ dch -i -------------------- Compile and install packages with debug symbols by the following. -------------------- $ export DEB_BUILD_OPTIONS=nostrip,noopt $ debuild $ cd .. $ sudo debi package_name*.changes -------------------- You need to check build scripts of the package and ensure to use "`CFLAGS=-g -Wall`" for compiling binaries. ==== Obtaining backtrace When you encounter program crash, reporting bug report with cut-and-pasted backtrace information is a good idea. The backtrace can be obtained by the following steps. - Run the program under `gdb`(1). - Reproduce crash. * It causes you to be dropped back to the `gdb` prompt. - Type "`bt`" at the `gdb` prompt. In case of program freeze, you can crash the program by pressing `Ctrl-C` in the terminal running `gdb` to obtain `gdb` prompt. TIP: Often, you see a backtrace where one or more of the top lines are in "`malloc()`" or "`g_malloc()`". When this happens, chances are your backtrace isn't very useful. The easiest way to find some useful information is to set the environment variable "`$MALLOC_CHECK_`" to a value of 2 (`malloc`(3)). You can do this while running `gdb` by doing the following. -------------------- $ MALLOC_CHECK_=2 gdb hello -------------------- ==== Advanced gdb commands .List of advanced gdb commands [grid="all"] `-----------------------------------`------------------------------------------------------------------------------ command description for command objectives ------------------------------------------------------------------------------------------------------------------- `(gdb) thread apply all bt` get a backtrace for all threads for multi-threaded program `(gdb) bt full` get parameters came on the stack of function calls `(gdb) thread apply all bt full` get a backtrace and parameters as the combination of the preceding options `(gdb) thread apply all bt full 10` get a backtrace and parameters for top 10 calls to cut off irrelevant output `(gdb) set logging on` write log of `gdb` output to a file (the default is "`gdb.txt`") ------------------------------------------------------------------------------------------------------------------- ==== Debugging X Errors If a GNOME program `preview1` has received an X error, you should see a message as follows. -------------------- The program 'preview1' received an X Window System error. -------------------- If this is the case, you can try running the program with "`--sync`", and break on the "`gdk_x_error`" function in order to obtain a backtrace. ==== Check dependency on libraries Use `ldd`(1) to find out a program@@@sq@@@s dependency on libraries by the followings. -------------------- $ ldd /bin/ls librt.so.1 => /lib/librt.so.1 (0x4001e000) libc.so.6 => /lib/libc.so.6 (0x40030000) libpthread.so.0 => /lib/libpthread.so.0 (0x40153000) /lib/ld-linux.so.2 => /lib/ld-linux.so.2 (0x40000000) -------------------- For `ls`(1) to work in a `chroot`ed environment, the above libraries must be available in your `chroot`ed environment. See <<_tracing_program_activities>>. ==== Memory leak detection tools There are several memory leak detection tools available in Debian. .List of memory leak detection tools [grid="all"] `----------------`-------------`------------`---------------------------------------------------- package popcon size description ------------------------------------------------------------------------------------------------- `libc6-dev` @-@popcon1@-@ @-@psize1@-@ `mtrace`(1): malloc debugging functionality in glibc `valgrind` @-@popcon1@-@ @-@psize1@-@ memory debugger and profiler `kmtrace` @-@popcon1@-@ @-@psize1@-@ KDE memory leak tracer using glibc@@@sq@@@s `mtrace`(1) `alleyoop` @-@popcon1@-@ @-@psize1@-@ GNOME front-end to the Valgrind memory checker `electric-fence` @-@popcon1@-@ @-@psize1@-@ `malloc`(3) debugger `leaktracer` @-@popcon1@-@ @-@psize1@-@ memory-leak tracer for C++ programs `libdmalloc5` @-@popcon1@-@ @-@psize1@-@ debug memory allocation library ------------------------------------------------------------------------------------------------- ==== Static code analysis tools There are http://en.wikipedia.org/wiki/Lint_programming_tool[lint] like tools for http://en.wikipedia.org/wiki/List_of_tools_for_static_code_analysis[static code analysis]. .List of tools for static code analysis [grid="all"] `---------------`-------------`------------`--------------------------------------------------------------------- package popcon size description ----------------------------------------------------------------------------------------------------------------- `splint` @-@popcon1@-@ @-@psize1@-@ tool for statically checking C programs for bugs `rats` @-@popcon1@-@ @-@psize1@-@ rough Auditing Tool for Security (C, C++, PHP, Perl, and Python code) `flawfinder` @-@popcon1@-@ @-@psize1@-@ tool to examine C/C++ source code and looks for security weaknesses `perl` @-@popcon1@-@ @-@psize1@-@ interpreter with internal static code checker: `B::Lint`(3perl) `pylint` @-@popcon1@-@ @-@psize1@-@ Python code static checker `jlint` @-@popcon1@-@ @-@psize1@-@ Java program checker `weblint-perl` @-@popcon1@-@ @-@psize1@-@ syntax and minimal style checker for HTML `linklint` @-@popcon1@-@ @-@psize1@-@ fast link checker and web site maintenance tool `libxml2-utils` @-@popcon1@-@ @-@psize1@-@ utilities with `xmllint`(1) to validate XML files ----------------------------------------------------------------------------------------------------------------- ==== Disassemble binary You can disassemble binary code with `objdump`(1) by the following. -------------------- $ objdump -m i386 -b binary -D /usr/lib/grub/x86_64-pc/stage1 -------------------- NOTE: `gdb`(1) may be used to disassemble code interactively. === Flex — a better Lex http://en.wikipedia.org/wiki/Flex_lexical_analyser[Flex] is a http://en.wikipedia.org/wiki/Lex_programming_tool[Lex]-compatible fast http://en.wikipedia.org/wiki/Lexical_analysis[lexical analyzer] generator. Tutorial for `flex`(1) can be found in "`info flex`". You need to provide your own "`main()`" and "`yywrap()`". Otherwise, your flex program should look like this to compile without a library. This is because that "`yywrap`" is a macro and "`%option main`" turns on "`%option noyywrap`" implicitly. -------------------- %option main %% .|\n ECHO ; %% -------------------- Alternatively, you may compile with the "`-lfl`" linker option at the end of your `cc`(1) command line (like AT&T-Lex with "`-ll`"). No "`%option`" is needed in this case. === Bison — a better Yacc Several packages provide a http://en.wikipedia.org/wiki/Yacc[Yacc]-compatible lookahead http://en.wikipedia.org/wiki/LR_parser[LR parser] or http://en.wikipedia.org/wiki/LALR_parser[LALR parser] generator in Debian. .List of Yacc-compatible LALR parser generators [grid="all"] `--------`-------------`------------`----------------------------------------------------------------- package popcon size description ------------------------------------------------------------------------------------------------------ `bison` @-@popcon1@-@ @-@psize1@-@ http://en.wikipedia.org/wiki/GNU_bison[GNU LALR parser generator] `byacc` @-@popcon1@-@ @-@psize1@-@ Berkeley LALR parser generator `btyacc` @-@popcon1@-@ @-@psize1@-@ backtracking parser generator based on `byacc` ------------------------------------------------------------------------------------------------------ Tutorial for `bison`(1) can be found in "`info bison`". You need to provide your own "`main()`" and "`yyerror()`". "`main()`" calls "`yyparse()`" which calls "`yylex()`", usually created with Flex. -------------------- %% %% -------------------- === Autoconf http://en.wikipedia.org/wiki/Autoconf[Autoconf] is a tool for producing shell scripts that automatically configure software source code packages to adapt to many kinds of Unix-like systems using the entire GNU build system. `autoconf`(1) produces the configuration script "`configure`". "`configure`" automatically creates a customized "`Makefile`" using the "`Makefile.in`" template. ==== Compile and install a program WARNING: Do not overwrite system files with your compiled programs when installing them. Debian does not touch files in "`/usr/local/`" or "`/opt`". So if you compile a program from source, install it into "`/usr/local/`" so it does not interfere with Debian. -------------------- $ cd src $ ./configure --prefix=/usr/local $ make $ make install # this puts the files in the system -------------------- ==== Uninstall program If you have the original source and if it uses `autoconf`(1)/`automake`(1) and if you can remember how you configured it, execute as follows to uninstall the program. -------------------- $ ./configure "all-of-the-options-you-gave-it" # make uninstall -------------------- Alternatively, if you are absolutely sure that the install process puts files only under "`/usr/local/`" and there is nothing important there, you can erase all its contents by the following. -------------------- # find /usr/local -type f -print0 | xargs -0 rm -f -------------------- If you are not sure where files are installed, you should consider using `checkinstall`(8) from the `checkinstall` package, which provides a clean path for the uninstall. It now supports to create a Debian package with "`-D`" option. === Perl short script madness Although any http://en.wikipedia.org/wiki/AWK[AWK] scripts can be automatically rewritten in http://en.wikipedia.org/wiki/Perl[Perl] using `a2p`(1), one-liner AWK scripts are best converted to one-liner Perl scripts manually. Let@@@sq@@@s think following AWK script snippet. -------------------- awk '($2=="1957") { print $3 }' | -------------------- This is equivalent to any one of the following lines. -------------------- perl -ne '@f=split; if ($f[1] eq "1957") { print "$f[2]\n"}' | -------------------- -------------------- perl -ne 'if ((@f=split)[1] eq "1957") { print "$f[2]\n"}' | -------------------- -------------------- perl -ne '@f=split; print $f[2] if ( $f[1]==1957 )' | -------------------- -------------------- perl -lane 'print $F[2] if $F[1] eq "1957"' | -------------------- -------------------- perl -lane 'print$F[2]if$F[1]eq+1957' | -------------------- The last one is a riddle. It took advantage of following Perl features. - The whitespace is optional. - The automatic conversion exists from number to the string. See `perlrun`(1) for the command-line options. For more crazy Perl scripts, http://perlgolf.sourceforge.net[Perl Golf] may be interesting. === Web Basic interactive dynamic web pages can be made as follows. - Queries are presented to the browser user using http://en.wikipedia.org/wiki/HTML[HTML] forms. - Filling and clicking on the form entries sends one of the following http://en.wikipedia.org/wiki/Uniform_Resource_Locator[URL] string with encoded parameters from the browser to the web server. * "`http://www.foo.dom/cgi-bin/program.pl?VAR1=VAL1&VAR2=VAL2&VAR3=VAL3`" * "`http://www.foo.dom/cgi-bin/program.py?VAR1=VAL1&VAR2=VAL2&VAR3=VAL3`" * "`http://www.foo.dom/program.php?VAR1=VAL1&VAR2=VAL2&VAR3=VAL3`" - "`%nn`" in URL is replaced with a character with hexadecimal `nn` value. - The environment variable is set as: "`QUERY_STRING="VAR1=VAL1 VAR2=VAL2 VAR3=VAL3"`". - http://en.wikipedia.org/wiki/Common_Gateway_Interface[CGI] program (any one of "`program.\*`") on the web server executes itself with the environment variable "`$QUERY_STRING`". - `stdout` of CGI program is sent to the web browser and is presented as an interactive dynamic web page. For security reasons it is better not to hand craft new hacks for parsing CGI parameters. There are established modules for them in Perl and Python. http://en.wikipedia.org/wiki/PHP[PHP] comes with these functionalities. When client data storage is needed, http://en.wikipedia.org/wiki/HTTP_cookie[HTTP cookies] are used. When client side data processing is needed, http://en.wikipedia.org/wiki/JavaScript[Javascript] is frequently used. For more, see the http://en.wikipedia.org/wiki/Common_Gateway_Interface[Common Gateway Interface], http://en.wikipedia.org/wiki/Apache_Software_Foundation[The Apache Software Foundation], and http://en.wikipedia.org/wiki/JavaScript[JavaScript]. Searching "CGI tutorial" on Google by typing encoded URL http://www.google.com/search?hl=en@@@amp@@@ie=UTF-8@@@amp@@@q=CGI+tutorial[http://www.google.com/search?hl=en@@@amp@@@ie=UTF-8@@@amp@@@q=CGI+tutorial] directly to the browser address is a good way to see the CGI script in action on the Google server. === The source code translation There are programs to convert source codes. .List of source code translation tools [grid="all"] `-----------`-------------`------------`----------`------------------------------------------------------------------ package popcon size keyword description --------------------------------------------------------------------------------------------------------------------- `perl` @-@popcon1@-@ @-@psize1@-@ AWK->PERL convert source codes from AWK to PERL: `a2p`(1) `f2c` @-@popcon1@-@ @-@psize1@-@ FORTRAN->C convert source codes from FORTRAN 77 to C/C++: `f2c`(1) `protoize` @-@popcon1@-@ @-@psize1@-@ ANSI C create/remove ANSI prototypes from C code `intel2gas` @-@popcon1@-@ @-@psize1@-@ intel->gas converter from NASM (Intel format) to the GNU Assembler (GAS) --------------------------------------------------------------------------------------------------------------------- === Making Debian package If you want to make a Debian package, read followings. - <<_debian_package_management>> to understand the basic package system - <<_porting_a_package_to_the_stable_system>> to understand basic porting process - <<_chroot_system>> to understand basic chroot techniques - `debuild`(1), `pbuilder`(1) and `pdebuild`(1) - <<_debugging_the_debian_package>> for recompiling for debugging - http://www.debian.org/doc/manuals/maint-guide/[Debian New Maintainers' Guide] as tutorial (the `maint-guide` package) - http://www.debian.org/doc/manuals/developers-reference/[Debian Developer@@@sq@@@s Reference] (the `developers-reference` package) - http://www.debian.org/doc/debian-policy/[Debian Policy Manual] (the `debian-policy` package) There are packages such as `dh-make`, `dh-make-perl`, etc., which help packaging. debian-reference-2.53/asciidoc/05_network.txt0000644000000000000000000022644412255345173016046 0ustar == Network setup // vim: set sts=2 expandtab: // Use ":set nowrap" to edit table TIP: For general guide to the GNU/Linux networking, read the http://www.tldp.org/LDP/nag2/[Linux Network Administrators Guide]. TIP: Although this document still uses old `ifconfig`(8) with IPv4 for its network configuration examples, Debian is moving to `ip`(8) with IPv4+IPv6 in the `wheezy` release. Patches to update this document are welcomed. === The basic network infrastructure Let's review the basic network infrastructure on the modern Debian system. .List of network configuration tools [grid="all"] `-----------------------------------`-------------`------------`-----------------`-------------------------------------------------------------------- packages popcon size type description ------------------------------------------------------------------------------------------------------------------------------------------------------ `ifupdown` @-@popcon1@-@ @-@psize1@-@ config::ifupdown standardized tool to bring up and down the network (Debian specific) `ifplugd` @-@popcon1@-@ @-@psize1@-@ , , manage the wired network automatically `ifupdown-extra` @-@popcon1@-@ @-@psize1@-@ , , network testing script to enhance "`ifupdown`" package `ifmetric` @-@popcon1@-@ @-@psize1@-@ , , set routing metrics for a network interface `guessnet` @-@popcon1@-@ @-@psize1@-@ , , mapping script to enhance "`ifupdown`" package via "`/etc/network/interfaces`" file `ifscheme` @-@popcon1@-@ @-@psize1@-@ , , mapping scripts to enhance "`ifupdown`" package `ifupdown-scripts-zg2` @-@popcon1@-@ @-@psize1@-@ , , Zugschlus' interface scripts for ifupdown@@@sq@@@s manual method `network-manager` @-@popcon1@-@ @-@psize1@-@ config::NM http://en.wikipedia.org/wiki/NetworkManager[NetworkManager] (daemon): manage the network automatically `network-manager-gnome` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/NetworkManager[NetworkManager] (GNOME frontend) `plasma-widget-networkmanagement` @-@popcon1@-@ @-@psize1@-@ , , http://en.wikipedia.org/wiki/NetworkManager[NetworkManager] (KDE frontend) `wicd` @-@popcon1@-@ @-@psize1@-@ config::wicd wired and wireless network manager (metapackage) `wicd-cli` @-@popcon1@-@ @-@psize1@-@ , , wired and wireless network manager (command-line client) `wicd-curses` @-@popcon1@-@ @-@psize1@-@ , , wired and wireless network manager (Curses client) `wicd-daemon` @-@popcon1@-@ @-@psize1@-@ , , wired and wireless network manager (daemon) `wicd-gtk` @-@popcon1@-@ @-@psize1@-@ , , wired and wireless network manager (GTK+ client) `iptables` @-@popcon1@-@ @-@psize1@-@ config::Netfilter administration tools for packet filtering and NAT (http://en.wikipedia.org/wiki/Netfilter[Netfilter]) `iproute` @-@popcon1@-@ @-@psize1@-@ config::iproute2 http://www.linuxfoundation.org/collaborate/workgroups/networking/iproute2[iproute2], IPv6 and other advanced network configuration: `ip`(8), `tc`(8), etc `ifrename` @-@popcon1@-@ @-@psize1@-@ , , rename network interfaces based on various static criteria: `ifrename`(8) `ethtool` @-@popcon1@-@ @-@psize1@-@ , , display or change Ethernet device settings `iputils-ping` @-@popcon1@-@ @-@psize1@-@ test::iproute2 test network reachability of a remote host by http://en.wikipedia.org/wiki/Hostname[hostname] or http://en.wikipedia.org/wiki/IP_address[IP address] (http://www.linuxfoundation.org/collaborate/workgroups/networking/iproute2[iproute2]) `iputils-arping` @-@popcon1@-@ @-@psize1@-@ , , test network reachability of a remote host specified by the http://en.wikipedia.org/wiki/Address_Resolution_Protocol[ARP] address `iputils-tracepath` @-@popcon1@-@ @-@psize1@-@ , , trace the network path to a remote host `net-tools` @-@popcon1@-@ @-@psize1@-@ config::net-tools NET-3 networking toolkit (http://www.linuxfoundation.org/collaborate/workgroups/networking/net-tools[net-tools], IPv4 network configuration): `ifconfig`(8) etc. `inetutils-ping` @-@popcon1@-@ @-@psize1@-@ test::net-tools test network reachability of a remote host by http://en.wikipedia.org/wiki/Hostname[hostname] or http://en.wikipedia.org/wiki/IP_address[IP address] (legacy, GNU) `arping` @-@popcon1@-@ @-@psize1@-@ , , test network reachability of a remote host specified by the http://en.wikipedia.org/wiki/Address_Resolution_Protocol[ARP] address (legacy) `traceroute` @-@popcon1@-@ @-@psize1@-@ , , trace the network path to a remote host (legacy, console) `isc-dhcp-client` @-@popcon1@-@ @-@psize1@-@ config::low-level DHCP client `wpasupplicant` @-@popcon1@-@ @-@psize1@-@ , , client support for WPA and WPA2 (IEEE 802.11i) `wpagui` @-@popcon1@-@ @-@psize1@-@ , , Qt GUI client for wpa_supplicant `wireless-tools` @-@popcon1@-@ @-@psize1@-@ , , tools for manipulating Linux Wireless Extensions `ppp` @-@popcon1@-@ @-@psize1@-@ , , PPP/PPPoE connection with `chat` `pppoeconf` @-@popcon1@-@ @-@psize1@-@ config::helper configuration helper for PPPoE connection `pppconfig` @-@popcon1@-@ @-@psize1@-@ , , configuration helper for PPP connection with `chat` `wvdial` @-@popcon1@-@ @-@psize1@-@ , , configuration helper for PPP connection with `wvdial` and `ppp` `mtr-tiny` @-@popcon1@-@ @-@psize1@-@ test::low-level trace the network path to a remote host (curses) `mtr` @-@popcon1@-@ @-@psize1@-@ , , trace the network path to a remote host (curses and GTK+) `gnome-nettool` @-@popcon1@-@ @-@psize1@-@ , , tools for common network information operations (GNOME) `nmap` @-@popcon1@-@ @-@psize1@-@ , , network mapper / port scanner (http://en.wikipedia.org/wiki/Nmap[Nmap], console) `zenmap` @-@popcon1@-@ @-@psize1@-@ , , network mapper / port scanner (GTK+) `tcpdump` @-@popcon1@-@ @-@psize1@-@ , , network traffic analyzer (http://en.wikipedia.org/wiki/Tcpdump[Tcpdump], console) `wireshark` @-@popcon1@-@ @-@psize1@-@ , , network traffic analyzer (http://en.wikipedia.org/wiki/Wireshark[Wireshark], GTK+) `tshark` @-@popcon1@-@ @-@psize1@-@ , , network traffic analyzer (console) `nagios3` @-@popcon1@-@ @-@psize1@-@ , , monitoring and management system for hosts, services and networks (http://en.wikipedia.org/wiki/Nagios[Nagios]) `tcptrace` @-@popcon1@-@ @-@psize1@-@ , , produce a summarization of the connections from `tcpdump` output `snort` @-@popcon1@-@ @-@psize1@-@ , , flexible network intrusion detection system (http://en.wikipedia.org/wiki/Snort_(software)[Snort]) `ntop` @-@popcon1@-@ @-@psize1@-@ , , display network usage in web browser `dnsutils` @-@popcon1@-@ @-@psize1@-@ , , network clients provided with http://en.wikipedia.org/wiki/BIND[BIND]: `nslookup`(8), `nsupdate`(8), `dig`(8) `dlint` @-@popcon1@-@ @-@psize1@-@ , , check http://en.wikipedia.org/wiki/Domain_Name_System[DNS] zone information using nameserver lookups `dnstracer` @-@popcon1@-@ @-@psize1@-@ , , trace a chain of http://en.wikipedia.org/wiki/Domain_Name_System[DNS] servers to the source ------------------------------------------------------------------------------------------------------------------------------------------------------ ==== The hostname resolution The hostname resolution is currently supported by the http://en.wikipedia.org/wiki/Name_Service_Switch[NSS (Name Service Switch)] mechanism too. The flow of this resolution is the following. 1. The "`/etc/nsswitch.conf`" file with stanza like "`hosts: files dns`" dictates the hostname resolution order. (This replaces the old functionality of the "`order`" stanza in "`/etc/host.conf`".) 2. The `files` method is invoked first. If the hostname is found in the "`/etc/hosts`" file, it returns all valid addresses for it and exits. (The "`/etc/host.conf`" file contains "`multi on`".) 3. The `dns` method is invoked. If the hostname is found by the query to the http://en.wikipedia.org/wiki/Domain_Name_System[Internet Domain Name System (DNS)] identified by the "`/etc/resolv.conf`" file, it returns all valid addresses for it and exits. For example, "`/etc/hosts`" looks like the following. -------------------- 127.0.0.1 localhost 127.0.1.1 # The following lines are desirable for IPv6 capable hosts ::1 ip6-localhost ip6-loopback fe00::0 ip6-localnet ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters ff02::3 ip6-allhosts -------------------- Each line starts with a http://en.wikipedia.org/wiki/IP_address[IP address] and it is followed by the associated http://en.wikipedia.org/wiki/Hostname[hostname]. The IP address `127.0.1.1` in the second line of this example may not be found on some other Unix-like systems. The http://en.wikipedia.org/wiki/Debian-Installer[Debian Installer] creates this entry for a system without a permanent IP address as a workaround for some software (e.g., GNOME) as documented in the http://bugs.debian.org/719621[bug #719621]. The matches the hostname defined in the "`/etc/hostname`". For a system with a permanent IP address, that permanent IP address should be used here instead of `127.0.1.1`. For a system with a permanent IP address and a http://en.wikipedia.org/wiki/FQDN[fully qualified domain name (FQDN)] provided by the http://en.wikipedia.org/wiki/Domain_Name_System[Domain Name System (DNS)], that canonical . should be used instead of just . The "`/etc/resolv.conf`" is a static file if the `resolvconf` package is not installed. If installed, it is a symbolic link. Either way, it contains information that initialize the resolver routines. If the DNS is found at IP="`192.168.11.1`", it contains the following. -------------------- nameserver 192.168.11.1 -------------------- The `resolvconf` package makes this "`/etc/resolv.conf`" into a symbolic link and manages its contents by the hook scripts automatically. For the PC workstation on the typical adhoc LAN environment, the hostname can be resolved via Multicast DNS (mDNS, http://en.wikipedia.org/wiki/Zeroconf[Zeroconf]) in addition to the basic `files` and `dns` methods. - http://en.wikipedia.org/wiki/Avahi_(software)[Avahi] provides a framework for Multicast DNS Service Discovery on Debian. - It is equivalent of http://en.wikipedia.org/wiki/Bonjour_(software)[Apple Bonjour / Apple Rendezvous]. - The `libnss-mdns` plugin package provides host name resolution via mDNS for the GNU Name Service Switch (NSS) functionality of the GNU C Library (glibc). - The "`/etc/nsswitch.conf`" file should have stanza like "`hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4`". - Host names ending with the http://en.wikipedia.org/wiki/.local[".local"] http://en.wikipedia.org/wiki/Pseudo-top-level_domain[pseudo-top-level domain] (TLD) are resolved. - The mDNS IPv4 link-local multicast address "`224.0.0.251`" or its IPv6 equivalent "`FF02::FB`" are used to make DNS query for a name ending with "`.local`". The hostname resolution via deprecated http://en.wikipedia.org/wiki/NetBIOS_over_TCP/IP[NETBios over TCP/IP] used by the older Windows system can be provided by installing the `winbind` package. The "`/etc/nsswitch.conf`" file should have stanza like "`hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4 wins`" to enable this functionality. (Modern Windows system usually use the `dns` method for the hostname resolution.) ==== The network interface name The network interface name, e.g. `eth0`, is assigned to each hardware in the Linux kernel through the user space configuration mechanism, `udev` (see <<_the_udev_system>>), as it is found. The network interface name is referred as **physical interface** in `ifup`(8) and `interfaces`(5). In order to ensure each network interface to be named persistently for each reboot using http://en.wikipedia.org/wiki/MAC_address[MAC address] etc., there is a rules file "`/etc/udev/rules.d/70-persistent-net.rules`". This file is automatically generated by the "`/lib/udev/write_net_rules`" program, probably run by the "`persistent-net-generator.rules`" rules file. You can modify it to change naming rule. CAUTION: When editing the "`/etc/udev/rules.d/70-persistent-net.rules`" rules file, you must keep each rule on a single line and the http://en.wikipedia.org/wiki/MAC_address[MAC address] in lowercase. For example, if you find "FireWire device" and "PCI device" in this file, you probably want to name "PCI device" as `eth0` and configure it as the primary network interface. ==== The network address range for the LAN Let us be reminded of the IPv4 32 bit address ranges in each class reserved for use on the http://en.wikipedia.org/wiki/Local_area_network[local area networks (LANs)] by http://tools.ietf.org/html/rfc1918[rfc1918]. These addresses are guaranteed not to conflict with any addresses on the Internet proper. .List of network address ranges [grid="all"] `-----`----------------------------`-------------`--------------`------------ Class network addresses net mask net mask /bits # of subnets ----------------------------------------------------------------------------- A 10.x.x.x 255.0.0.0 /8 1 B 172.16.x.x -- 172.31.x.x 255.255.0.0 /16 16 C 192.168.0.x -- 192.168.255.x 255.255.255.0 /24 256 ----------------------------------------------------------------------------- NOTE: If one of these addresses is assigned to a host, then that host must not access the Internet directly but must access it through a gateway that acts as a proxy for individual services or else does http://en.wikipedia.org/wiki/Network_address_translation[Network Address Translation (NAT)]. The broadband router usually performs NAT for the consumer LAN environment. ==== The network device support Although most hardware devices are supported by the Debian system, there are some network devices which require http://www.debian.org/social_contract#guidelines[DFSG] non-free firmware to support them. Please see <<_hardware_drivers_and_firmware>>. === The modern network configuration for desktop Debian `squeeze` and newer can manage the network connection via management http://en.wikipedia.org/wiki/Daemon_(computer_software)[daemon] software such as http://en.wikipedia.org/wiki/NetworkManager[NetworkManager (NM)] (network-manager and associated packages) or http://en.wikipedia.org/wiki/Wicd_(Linux_Network_Manager)[Wicd] (wicd and associated packages). - They come with their own http://en.wikipedia.org/wiki/Graphical_user_interface[GUI] and command-line programs as their user interfaces. - They come with their own http://en.wikipedia.org/wiki/Daemon_(computer_software)[daemon] as their backend system. - They allow easy connection of your system to the Internet. - They allow easy management of wired and wireless network configuration. - They allow us to configure network independent of the legacy `ifupdown` package. NOTE: Do not use these automatic network configuration tools for servers. These are aimed primarily for mobile desktop users on laptops. These modern network configuration tools need to be configured properly to avoid conflicting with the legacy `ifupdown` package and its configuration file "`/etc/network/interfaces`". NOTE: Some features of these automatic network configuration tools may suffer regressions. These are not as robust as the legacy `ifupdown` package. Check http://bugs.debian.org/cgi-bin/pkgreport.cgi?package=network-manager[BTS of network-manager] and http://bugs.debian.org/cgi-bin/pkgreport.cgi?package=wicd[BTS of wicd] for current issues and limitations. ==== GUI network configuration tools Official documentations for NM and Wicd on Debian are provided in "`/usr/share/doc/network-manager/README.Debian`" and "`/usr/share/doc/wicd/README.Debian`", respectively. Essentially, the network configuration for desktop is done as follows. 1. Make desktop user, e.g. `foo`, belong to group "`netdev`" by the following (Alternatively, do it automatically via http://en.wikipedia.org/wiki/D-Bus[D-bus] under modern desktop environments such as GNOME and KDE). + -------------------- $ sudo adduser foo netdev -------------------- 2. Keep configuration of "`/etc/network/interfaces`" as simple as in the following. + -------------------- auto lo iface lo inet loopback -------------------- 3. Restart NM or Wicd by the following. + -------------------- $ sudo /etc/init.d/network-manager restart -------------------- + -------------------- $ sudo /etc/init.d/wicd restart -------------------- 4. Configure your network via GUI. NOTE: Only interfaces which are **not** listed in "`/etc/network/interfaces`" are managed by NM or Wicd to avoid conflict with `ifupdown`. TIP: If you wish to extend network configuration capabilities of NM, please seek appropriate plug-in modules and supplemental packages such as `network-manager-openconnect`, `network-manager-openvpn-gnome`, `network-manager-pptp-gnome`, `mobile-broadband-provider-info`, `gnome-bluetooth`, etc. The same goes for those of Wicd. CAUTION: These automatic network configuration tools may not be compatible with esoteric configurations of legacy `ifupdown` in "`/etc/network/interfaces`" such as ones in <<_the_basic_network_configuration_with_ifupdown_legacy>> and <<_the_advanced_network_configuration_with_ifupdown_legacy>>. Check http://bugs.debian.org/cgi-bin/pkgreport.cgi?package=network-manager[BTS of network-manager] and http://bugs.debian.org/cgi-bin/pkgreport.cgi?package=wicd[BTS of wicd] for current issues and limitations. === The legacy network connection and configuration When the method described in <<_the_modern_network_configuration_for_desktop>> does not suffice your needs, you should use the legacy network connection and configuration method which combines many simpler tools. The legacy network connection is specific for each method (see <<_the_network_connection_method_legacy>>). There are 2 types of programs for the low level network configuration on Linux (see <<_iproute2_commands>>). - Old http://www.linuxfoundation.org/collaborate/workgroups/networking/net-tools[net-tools] programs (`ifconfig`(8), ...) are from the Linux NET-3 networking system. Most of these are obsolete now. - New http://www.linuxfoundation.org/collaborate/workgroups/networking/iproute2[Linux iproute2] programs (`ip`(8), ...) are the current Linux networking system. Although these low level networking programs are powerful, they are cumbersome to use. So high level network configuration systems have been created. The `ifupdown` package is the de facto standard for such high level network configuration system on Debian. It enables you to bring up network simply by doing , e.g., "`ifup eth0`". Its configuration file is the "`/etc/network/interfaces`" file and its typical contents are the following. -------------------- auto lo iface lo inet loopback auto eth0 iface eth0 inet dhcp -------------------- The `resolvconf` package was created to supplement `ifupdown` system to support smooth reconfiguration of network address resolution by automating rewrite of resolver configuration file "`/etc/resolv.conf`". Now, most Debian network configuration packages are modified to use `resolvconf` package (see "`/usr/share/doc/resolvconf/README.Debian`"). Helper scripts to the `ifupdown` package such as `ifplugd`, `guessnet`, `ifscheme`, etc. are created to automate dynamic configuration of network environment such as one for mobile PC on wired LAN. These are relatively difficult to use but play well with existing `ifupdown` system. These are explained in detail with examples (see <<_the_basic_network_configuration_with_ifupdown_legacy>> and <<_the_advanced_network_configuration_with_ifupdown_legacy>>). === The network connection method (legacy) CAUTION: The connection test methods described in this section are meant for testing purposes. It is not meant to be used directly for the daily network connection. You are advised to use NM, Wicd, or the `ifupdown` package instead (see <<_the_modern_network_configuration_for_desktop>> and <<_the_basic_network_configuration_with_ifupdown_legacy>>). // FIXME: Since my MacBook has no build-in modem, and I use the broadband-modem via Ethernet, all POTS-related description needs to be tested with the recent system. The typical network connection method and connection path for a PC can be summarized as the following. .List of network connection methods and connection paths [grid="all"] `----------------------`-----------------`------------------------------------------------------------------------------------------------ PC connection method connection path ------------------------------------------------------------------------------------------------------------------------------------------ Serial port (`ppp0`) PPP <=> http://en.wikipedia.org/wiki/Modem[modem] <=> POTS <=> dial-up access point <=> ISP Ethernet port (`eth0`) PPPoE/DHCP/Static <=> BB-modem <=> BB service <=> BB access point <=> ISP Ethernet port (`eth0`) DHCP/Static <=> LAN <=> BB-router with http://en.wikipedia.org/wiki/Network_address_translation[network address translation (NAT)] (<=> BB-modem ...) ------------------------------------------------------------------------------------------------------------------------------------------ Here is the summary of configuration scripts for each connection method. .List of network connection configurations [grid="all"] `-----------------`-----------------------------------------`------------------ connection method configuration backend package(s) ------------------------------------------------------------------------------- PPP `pppconfig` to create deterministic chat `pppconfig`, `ppp` PPP (alternative) `wvdialconf` to create heuristic chat `ppp`, `wvdial` PPPoE `pppoeconf` to create deterministic chat `pppoeconf`, `ppp` DHCP described in "`/etc/dhcp/dhclient.conf`" `isc-dhcp-client` static IP (IPv4) described in "`/etc/network/interfaces`" `iproute` or `net-tools` (obsolete) static IP (IPv6) described in "`/etc/network/interfaces`" `iproute` ------------------------------------------------------------------------------- The network connection acronyms mean the following. .List of network connection acronyms [grid="all"] `-------------------------------------------------------------------------`--------------------------------------------------------------- acronym meaning ------------------------------------------------------------------------------------------------------------------------------------------ http://en.wikipedia.org/wiki/Plain_old_telephone_service[POTS] plain old telephone service BB http://en.wikipedia.org/wiki/Broadband[broadband] BB-service e.g., the digital subscriber line (DSL), the cable TV, or the fiber to the premises (FTTP) BB-modem e.g., http://en.wikipedia.org/wiki/DSL_modem[the DSL modem], http://en.wikipedia.org/wiki/Cable_modem[the cable modem], or http://en.wikipedia.org/wiki/FTTP[the optical network terminal (ONT)] http://en.wikipedia.org/wiki/Local_area_network[LAN] local area network http://en.wikipedia.org/wiki/Wide_area_network[WAN] wide area network http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol[DHCP] dynamic host configuration protocol http://en.wikipedia.org/wiki/Point-to-Point_Protocol[PPP] point-to-point protocol http://en.wikipedia.org/wiki/Point-to-Point_Protocol_over_Ethernet[PPPoE] point-to-point protocol over Ethernet http://en.wikipedia.org/wiki/ISP[ISP] Internet service provider ------------------------------------------------------------------------------------------------------------------------------------------ NOTE: The WAN connection services via cable TV are generally served by DHCP or PPPoE. The ones by ADSL and FTTP are generally served by PPPoE. You have to consult your ISP for exact configuration requirements of the WAN connection. NOTE: When BB-router is used to create home LAN environment, PCs on LAN are connected to the WAN via BB-router with http://en.wikipedia.org/wiki/Network_address_translation[network address translation (NAT)]. For such case, PC@@@sq@@@s network interfaces on the LAN are served by static IP or DHCP from the BB-router. BB-router must be configured to connect the WAN following the instruction by your ISP. ==== The DHCP connection with the Ethernet The typical modern home and small business network, i.e. LAN, are connected to the WAN (Internet) using some consumer grade broadband router. The LAN behind this router is usually served by the http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol[dynamic host configuration protocol (DHCP)] server running on the router. Just install the `isc-dhcp-client` package for the Ethernet served by the http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol[dynamic host configuration protocol (DHCP)]. See `dhclient.conf`(5). ==== The static IP connection with the Ethernet No special action is needed for the Ethernet served by the static IP. ==== The PPP connection with pppconfig The configuration script `pppconfig` configures the http://en.wikipedia.org/wiki/Point-to-Point_Protocol[PPP] connection interactively just by selecting the following. - The telephone number - The ISP user name - The ISP password - The port speed - The modem communication port - The authentication method .List of configuration files for the http://en.wikipedia.org/wiki/Point-to-Point_Protocol[PPP] connection with pppconfig [grid="all"] `-----------------------------`----------------------------------------------------------------------------------------------------------- file function ------------------------------------------------------------------------------------------------------------------------------------------ `/etc/ppp/peers/` The `pppconfig` generated configuration file for `pppd` specific to `/etc/chatscripts/` The `pppconfig` generated configuration file for `chat` specific to `/etc/ppp/options` The general execution parameter for `pppd` `/etc/ppp/pap-secret` Authentication data for the http://en.wikipedia.org/wiki/Password_authentication_protocol[PAP] (security risk) `/etc/ppp/chap-secret` Authentication data for the http://en.wikipedia.org/wiki/Challenge-handshake_authentication_protocol[CHAP] (more secure) ------------------------------------------------------------------------------------------------------------------------------------------ CAUTION: The "" value of "`provider`" is assumed if `pon` and `poff` commands are invoked without arguments. You can test configuration using low level network configuration tools as the following. -------------------- $ sudo pon ... $ sudo poff -------------------- See "`/usr/share/doc/ppp/README.Debian.gz`". ==== The alternative PPP connection with wvdialconf A different approach to using `pppd`(8) is to run it from `wvdial`(1) which comes in the `wvdial` package. Instead of `pppd` running `chat`(8) to dial in and negotiate the connection, `wvdial` does the dialing and initial negotiating and then starts `pppd` to do the rest. The configuration script `wvdialconf` configures the PPP connection interactively just by selecting the following. - The telephone number - The ISP user name - The ISP password `wvdial` succeeds in making the connection in most cases and maintains authentication data list automatically. .List of configuration files for the PPP connection with wvdialconf [grid="all"] `-----------------------`------------------------------------------------------------------------------------------------------------------ file function ------------------------------------------------------------------------------------------------------------------------------------------ `/etc/ppp/peers/wvdial` The `wvdialconf` generated configuration file for `pppd` specific to `wvdial` `/etc/wvdial.conf` The `wvdialconf` generated configuration file `/etc/ppp/options` The general execution parameter for `pppd` `/etc/ppp/pap-secret` Authentication data for the http://en.wikipedia.org/wiki/Password_authentication_protocol[PAP] (security risk) `/etc/ppp/chap-secret` Authentication data for the http://en.wikipedia.org/wiki/Challenge-handshake_authentication_protocol[CHAP] (more secure) ------------------------------------------------------------------------------------------------------------------------------------------ You can test configuration using low level network configuration tools as the following. -------------------- $ sudo wvdial ... $ sudo killall wvdial -------------------- See `wvdial`(1) and `wvdial.conf`(5). ==== The PPPoE connection with pppoeconf When your ISP serves you with PPPoE connection and you decide to connect your PC directly to the WAN, the network of your PC must be configured with the PPPoE. The PPPoE stand for PPP over Ethernet. The configuration script `pppoeconf` configures the PPPoE connection interactively. The configuration files are the following. .List of configuration files for the PPPoE connection with pppoeconf [grid="all"] `-----------------------------`----------------------------------------------------------------------------------------------------------- file function ------------------------------------------------------------------------------------------------------------------------------------------ `/etc/ppp/peers/dsl-provider` The `pppoeconf` generated configuration file for `pppd` specific to `pppoe` `/etc/ppp/options` The general execution parameter for `pppd` `/etc/ppp/pap-secret` Authentication data for the http://en.wikipedia.org/wiki/Password_authentication_protocol[PAP] (security risk) `/etc/ppp/chap-secret` Authentication data for the http://en.wikipedia.org/wiki/Challenge-handshake_authentication_protocol[CHAP] (more secure) ------------------------------------------------------------------------------------------------------------------------------------------ You can test configuration using low level network configuration tools as the following. -------------------- $ sudo /sbin/ifconfig eth0 up $ sudo pon dsl-provider ... $ sudo poff dsl-provider $ sudo /sbin/ifconfig eth0 down -------------------- See "`/usr/share/doc/pppoeconf/README.Debian`". === The basic network configuration with ifupdown (legacy) The traditional http://en.wikipedia.org/wiki/Internet_Protocol_Suite[TCP/IP network] setup on the Debian system uses `ifupdown` package as a high level tool. There are 2 typical cases. - For **dynamic IP** system such as mobile PCs, you should setup TCP/IP network **with** the `resolvconf` package and enable you to switch your network configuration easily (see <<_the_network_interface_served_by_the_dhcp>>). - For **static IP** system such as servers, you should setup TCP/IP network **without** the `resolvconf` package and keep your system simple (see <<_the_network_interface_with_the_static_ip>>). These traditional setup methods are quite useful if you wish to set up advanced configuration; find details in the following. The `ifupdown` package provides the standardized framework for the high level network configuration in the Debian system. In this section, we learn the basic network configuration with `ifupdown` with simplified introduction and many typical examples. ==== The command syntax simplified The `ifupdown` package contains 2 commands: `ifup`(8) and `ifdown`(8). They offer high level network configuration dictated by the configuration file "/etc/network/interfaces". .List of basic network configuration commands with ifupdown [grid="all"] `-------------`------------------------------------------------------------------------------------------------------- command action ---------------------------------------------------------------------------------------------------------------------- `ifup eth0` bring up a network interface `eth0` with the configuration `eth0` if "`iface eth0`" stanza exists `ifdown eth0` bring down a network interface `eth0` with the configuration `eth0` if "`iface eth0`" stanza exists ---------------------------------------------------------------------------------------------------------------------- WARNING: Do not use low level configuration tools such as `ifconfig`(8) and `ip`(8) commands to configure an interface in **up** state. NOTE: There is no command `ifupdown`. ==== The basic syntax of "/etc/network/interfaces" The key syntax of "`/etc/network/interfaces`" as explained in `interfaces`(5) can be summarized as the following. [[list-of-stanzas-in-eni]] .List of stanzas in "`/etc/network/interfaces`" [grid="all"] `-----------------------------------------------------`----------------------------------------------------------------------------------- stanza meaning ------------------------------------------------------------------------------------------------------------------------------------------ "`auto `" start interface upon start of the system "`allow-auto `" , , "`allow-hotplug `" start interface when the kernel detects a hotplug event from the interface Lines started with "`iface ...`" define the network configuration Lines started with "`mapping `" define mapping value of for the matching A line starting with a hash "`#`" ignore as comments (end-of-line comments are **not** supported) A line ending with a backslash "`\`" extend the configuration to the next line ------------------------------------------------------------------------------------------------------------------------------------------ Lines started with **`iface`** stanza has the following syntax. -------------------- iface ... -------------------- For the basic configuration, the **`mapping`** stanza is not used and you use the network interface name as the network configuration name (See <<_the_mapping_stanza>>). WARNING: Do not define duplicates of the "`iface`" stanza for a network interface in "`/etc/network/interfaces`". ==== The loopback network interface The following configuration entry in the "`/etc/network/interfaces`" file brings up the loopback network interface `lo` upon booting the system (via **`auto`** stanza). -------------------- auto lo iface lo inet loopback -------------------- This one always exists in the "`/etc/network/interfaces`" file. ==== The network interface served by the DHCP After preparing the system by <<_the_dhcp_connection_with_the_ethernet>>, the network interface served by the DHCP is configured by creating the configuration entry in the "`/etc/network/interfaces`" file as the following. -------------------- allow-hotplug eth0 iface eth0 inet dhcp -------------------- When the Linux kernel detects the physical interface `eth0`, the **`allow-hotplug`** stanza causes `ifup` to bring up the interface and the **`iface`** stanza causes `ifup` to use DHCP to configure the interface. ==== The network interface with the static IP The network interface served by the static IP is configured by creating the configuration entry in the "`/etc/network/interfaces`" file as the following. -------------------- allow-hotplug eth0 iface eth0 inet static address 192.168.11.100 netmask 255.255.255.0 gateway 192.168.11.1 dns-domain example.com dns-nameservers 192.168.11.1 -------------------- When the Linux kernel detects the physical interface `eth0`, the **`allow-hotplug`** stanza causes `ifup` to bring up the interface and the **`iface`** stanza causes `ifup` to use the static IP to configure the interface. Here, I assumed the following. - IP address range of the LAN network: `192.168.11.0` - `192.168.11.255` - IP address of the gateway: `192.168.11.1` - IP address of the PC: `192.168.11.100` - The `resolvconf` package: installed - The domain name: "`example.com`" - IP address of the DNS server: `192.168.11.1` When the `resolvconf` package is not installed, DNS related configuration needs to be done manually by editing the "`/etc/resolv.conf`" as the following. -------------------- nameserver 192.168.11.1 domain example.com -------------------- CAUTION: The IP addresses used in the above example are not meant to be copied literally. You have to adjust IP numbers to your actual network configuration. ==== The basics of wireless LAN interface The http://en.wikipedia.org/wiki/Wireless_LAN[wireless LAN (WLAN for short)] provides the fast wireless connectivity through the spread-spectrum communication of unlicensed radio bands based on the set of standards called http://en.wikipedia.org/wiki/IEEE_802.11[IEEE 802.11]. The WLAN interfaces are almost like normal Ethernet interfaces but require some network ID and encryption key data to be provided when they are initialized. Their high level network tools are exactly the same as that of Ethernet interfaces except interface names are a bit different like `eth1`, `wlan0`, `ath0`, `wifi0`, ... depending on the kernel drivers used. TIP: The `wmaster0` device is the master device which is an internal device used only by http://en.wikipedia.org/wiki/SoftMAC[SoftMAC] with new http://linuxwireless.org/[mac80211 API of Linux]. Here are some keywords to remember for the WLAN. .List of acronyms for WLAN [grid="all"] `-----------`--------------------------------------------------------------------------------------`--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- acronym full word meaning ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- NWID Network ID 16 bit network ID used by pre-802.11 http://en.wikipedia.org/wiki/WaveLAN[WaveLAN] network (very deprecated) (E)SSID (Extended) http://en.wikipedia.org/wiki/Service_set_identifier[Service Set Identifier] network name of the http://en.wikipedia.org/wiki/Wireless_access_point[Wireless Access Points (APs)] interconnected to form an integrated http://en.wikipedia.org/wiki/IEEE_802.11[802.11 wireless LAN], Domain ID WEP, (WEP2) http://en.wikipedia.org/wiki/Wired_Equivalent_Privacy[Wired Equivalent Privacy] 1st generation 64-bit (128-bit) wireless encryption standard with 40-bit key (deprecated) WPA http://en.wikipedia.org/wiki/Wi-Fi_Protected_Access[Wi-Fi Protected Access] 2nd generation wireless encryption standard (most of 802.11i), compatible with WEP WPA2 http://en.wikipedia.org/wiki/IEEE_802.11i[Wi-Fi Protected Access 2] 3rd generation wireless encryption standard (full 802.11i), non-compatible with WEP ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- The actual choice of protocol is usually limited by the wireless router you deploy. ==== The wireless LAN interface with WPA/WPA2 You need to install the `wpasupplicant` package to support the WLAN with the new WPA/WPA2. In case of the http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol[DHCP] served IP on WLAN connection, the "`/etc/network/interfaces`" file entry should be as the following. -------------------- allow-hotplug ath0 iface ath0 inet dhcp wpa-ssid homezone # hexadecimal psk is encoded from a plaintext passphrase wpa-psk 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f -------------------- See "`/usr/share/doc/wpasupplicant/README.modes.gz`". ==== The wireless LAN interface with WEP You need to install the `wireless-tools` package to support the WLAN with the old WEP. (Your consumer grade router may still be using this insecure infrastructure but this is better than nothing.) CAUTION: Please note that your network traffic on WLAN with WEP may be sniffed by others. In case of the http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol[DHCP] served IP on WLAN connection, the "`/etc/network/interfaces`" file entry should be as the following. -------------------- allow-hotplug eth0 iface eth0 inet dhcp wireless-essid Home wireless-key1 0123-4567-89ab-cdef wireless-key2 12345678 wireless-key3 s:password wireless-defaultkey 2 wireless-keymode open -------------------- See "`/usr/share/doc/wireless-tools/README.Debian`". ==== The PPP connection You need to configure the PPP connection first as described before (see <<_the_ppp_connection_with_pppconfig>>). Then, add the "`/etc/network/interfaces`" file entry for the primary PPP device `ppp0` as the following. -------------------- iface ppp0 inet ppp provider -------------------- ==== The alternative PPP connection You need to configure the alternative PPP connection with `wvdial` first as described before (see <<_the_alternative_ppp_connection_with_wvdialconf>>). Then, add the "`/etc/network/interfaces`" file entry for the primary PPP device `ppp0` as the following. -------------------- iface ppp0 inet wvdial -------------------- ==== The PPPoE connection For PC connected directly to the WAN served by the PPPoE, you need to configure system with the PPPoE connection as described before (see <<_the_pppoe_connection_with_pppoeconf>>). Then, add the "`/etc/network/interfaces`" file entry for the primary PPPoE device `eth0` as the following. -------------------- allow-hotplug eth0 iface eth0 inet manual pre-up /sbin/ifconfig eth0 up up ifup ppp0=dsl down ifdown ppp0=dsl post-down /sbin/ifconfig eth0 down # The following is used internally only iface dsl inet ppp provider dsl-provider -------------------- ==== The network configuration state of ifupdown The "`/etc/network/run/ifstate`" file stores the **intended** network configuration states for all the currently active network interfaces managed by the `ifupdown` package. Unfortunately, even if the `ifupdown` system fails to bring up the interface as intended, the "`/etc/network/run/ifstate`" file lists it active. Unless the output of the `ifconfig`(8) command for an interface does not have a line like following example, it can not be used as a part of http://en.wikipedia.org/wiki/IPv4[IPV4 network]. -------------------- inet addr:192.168.11.2 Bcast:192.168.11.255 Mask:255.255.255.0 -------------------- NOTE: For the Ethernet device connected to the PPPoE, the output of the `ifconfig`(8) command lacks a line which looks like above example. ==== The basic network reconfiguration When you try to reconfigure the interface, e.g. `eth0`, you must disable it first with the "**`sudo ifdown eth0`**" command. This removes the entry of `eth0` from the "`/etc/network/run/ifstate`" file. (This may result in some error message if `eth0` is not active or it is configured improperly previously. So far, it seems to be safe to do this for the simple single user work station at any time.) You are now free to rewrite the "`/etc/network/interfaces`" contents as needed to reconfigure the network interface, `eth0`. Then, you can reactivate `eth0` with the "**`sudo ifup eth0`**" command. TIP: You can (re)initialize the network interface simply by "**`sudo ifdown eth0;sudo ifup eth0`**". ==== The ifupdown-extra package The `ifupdown-extra` package provides easy network connection tests for use with the `ifupdown` package. - The `network-test`(1) command can be used from the shell. - The automatic scripts are run for each `ifup` command execution. The `network-test` command frees you from the execution of cumbersome low level commands to analyze the network problem. The automatic scripts are installed in "`/etc/network/\*/`" and perform the following. - Check the network cable connection - Check duplicate use of IP address - Setup system@@@sq@@@s static routes based on the "`/etc/network/routes`" definition - Check if network gateway is reachable - Record results in the "`/var/log/syslog`" file This syslog record is quite useful for administration of the network problem on the remote system. TIP: The automatic behavior of the `ifupdown-extra` package is configurable with the "`/etc/default/network-test`". Some of these automatic checks slow down the system boot-up a little bit since it takes some time to listen for http://en.wikipedia.org/wiki/Address_Resolution_Protocol[ARP] replies. === The advanced network configuration with ifupdown (legacy) The functionality of the `ifupdown` package can be improved beyond what was described in <<_the_basic_network_configuration_with_ifupdown_legacy>> with the advanced knowledge. The functionalities described here are completely optional. I, being lazy and minimalist, rarely bother to use these. CAUTION: If you could not set up network connection by information in <<_the_basic_network_configuration_with_ifupdown_legacy>>, you make situation worse by using information below. ==== The ifplugd package The `ifplugd` package is an older automatic network configuration tool which can manage only Ethernet connections. This solves unplugged/replugged Ethernet cable issues for mobile PC etc. If you have http://en.wikipedia.org/wiki/NetworkManager[NetworkManager] or http://en.wikipedia.org/wiki/Wicd_(Linux_Network_Manager)[Wicd] (see <<_the_modern_network_configuration_for_desktop>>) installed, you do not need this package. This package runs a http://en.wikipedia.org/wiki/Daemon_(computer_software)[daemon] and replaces **auto** or **allow-hotplug** functionalities (see <>) and starts interfaces upon their connection to the network. Here is how to use the `ifplugd` package for the internal Ethernet port, e.g. `eth0`. 1. Remove stanza in "`/etc/network/interfaces`": "`auto eth0`" or "`allow-hotplug eth0`". 2. Keep stanza in "`/etc/network/interfaces`": "`iface eth0 inet ...`" and "`mapping ...`". 3. Install the `ifplugd` package. 4. Run "`sudo dpkg-reconfigure ifplugd`". 5. Put `eth0` as the "static interfaces to be watched by ifplugd". Now, the network reconfiguration works as you desire. - Upon power-on or upon hardware discovery, the interface is not brought up by itself. * Quick boot process without the long DHCP timeout. * No funny activated interface without proper IPv4 address (see <<_the_network_configuration_state_of_ifupdown>>). - Upon finding the Ethernet cable, the interface is brought up. - Upon some time after unplugging the Ethernet cable, the interface is brought down automatically. - Upon plugging in another Ethernet cable, the interface is brought up under the new network environment. TIP: The arguments for the `ifplugd`(8) command can set its behaviors such as the delay for reconfiguring interfaces. ==== The ifmetric package The `ifmetric` package enables us to manipulate metrics of routes a posteriori even for DHCP. The following sets the `eth0` interface to be preferred over the `wlan0` interface. 1. Install the `ifmetric` package. 2. Add an option line with "`metric 0`" just below the "`iface eth0 inet dhcp`" line in "`/etc/network/interfaces`". 3. Add an option line with "`metric 1`" just below the "`iface wlan0 inet dhcp`" line in "`/etc/network/interfaces`". The metric 0 means the highest priority route and is the default one. The larger metric value means lower priority routes. The IP address of the active interface with the lowest metric value becomes the originating one. See `ifmetric`(8). ==== The virtual interface A single physical Ethernet interface can be configured as multiple virtual interfaces with different IP addresses. Usually the purpose is to connect an interface to several IP subnetworks. For example, IP address based virtual web hosting by a single network interface is one such application. For example, let@@@sq@@@s suppose the following. - A single Ethernet interface on your host is connected to a Ethernet hub (not to the broadband router). - The Ethernet hub is connected to both the Internet and LAN network. - The LAN network uses subnet `192.168.0.x/24`. - Your host uses DHCP served IP address with the physical interface `eth0` for the Internet. - Your host uses `192.168.0.1` with the virtual interface `eth0:0` for the LAN. The following stanzas in "`/etc/network/interfaces`" configure your network. -------------------- iface eth0 inet dhcp metric 0 iface eth0:0 inet static address 192.168.0.1 netmask 255.255.255.0 network 192.168.0.0 metric 1 -------------------- CAUTION: Although this configuration example with http://en.wikipedia.org/wiki/Network_address_translation[network address translation (NAT)] using http://en.wikipedia.org/wiki/Netfilter[netfilter/iptables] (see <<_netfilter_infrastructure>>) can provide cheap router for the LAN with only single interface, there is no real firewall capability with such set up. You should use 2 physical interfaces with NAT to secure the local network from the Internet. ==== The advanced command syntax The `ifupdown` package offers advanced network configuration using the **network configuration** name and the **network interface** name. I use a terminology being slightly different from the one used in `ifup`(8) and `interfaces`(5). .List of terminology for network devices [grid="all"] `---------------------------`------------------------------`-------------------------------------`---------------------------------------- manpage terminology my terminology examples in the following text description ------------------------------------------------------------------------------------------------------------------------------------------ **physical interface** name **network interface** name `lo`, `eth0`, `` name given by the Linux kernel (using `udev` mechanism) **logical interface** name **network configuration** name `config1`, `config2`, `` name token following **`iface`** in the "`/etc/network/interfaces`" ------------------------------------------------------------------------------------------------------------------------------------------ Basic network configuration commands in <<_the_command_syntax_simplified>> require the **network configuration** name token of the **`iface`** stanza to match the **network interface** name in the "`/etc/network/interfaces`". Advanced network configuration commands enables separation of the **network configuration** name and the **network interface** name in the "`/etc/network/interfaces`" as the following. .List of advanced network configuration commands with ifupdown [grid="all"] `---------------------`------------------------------------------------------------------------------------------------- command action ------------------------------------------------------------------------------------------------------------------------ `ifup eth0=config1` bring up a network interface `eth0` with the configuration `config1` `ifdown eth0=config1` bring down a network interface `eth0` with the configuration `config1` `ifup eth0` bring up a network interface `eth0` with the configuration selected by **`mapping`** stanza `ifdown eth0` bring down a network interface `eth0` with the configuration selected by **`mapping`** stanza ------------------------------------------------------------------------------------------------------------------------ ==== The mapping stanza We skipped explaining the **`mapping`** stanza in the "`/etc/network/interfaces`" in <<_the_basic_syntax_of_etc_network_interfaces>> to avoid complication. This stanza has the following syntax. -------------------- mapping script map map map ... -------------------- This provides advanced features to the "`/etc/network/interfaces`" file by automating the choice of the configuration with the mapping script specified by ``. Let@@@sq@@@s follow the execution of the following. -------------------- $ sudo ifup eth0 -------------------- When the "``" matches "`eth0`", this execution produces the execution of the following command to configure `eth0` automatically. -------------------- $ sudo ifup eth0=$(echo -e ' \n \n ...' | eth0) -------------------- Here, script input lines with "`map`" are optional and can be repeated. NOTE: The glob for **`mapping`** stanza works like shell filename glob (see <<_shell_glob>>). ==== The manually switchable network configuration Here is how to switch manually among several network configurations without rewriting the "`/etc/network/interfaces`" file as in <<_the_basic_network_reconfiguration>> . For all the network configuration you need to access, you create a separate stanza in "`/etc/network/interfaces`" file as the following. -------------------- auto lo iface lo inet loopback iface config1 inet dhcp iface config2 inet static address 192.168.11.100 netmask 255.255.255.0 gateway 192.168.11.1 dns-domain example.com dns-nameservers 192.168.11.1 iface pppoe inet manual pre-up /sbin/ifconfig eth0 up up ifup ppp0=dsl down ifdown ppp0=dsl post-down /sbin/ifconfig eth0 down # The following is used internally only iface dsl inet ppp provider dsl-provider iface pots inet ppp provider provider -------------------- Please note the **network configuration name** which is the token after **`iface`** does not use the token for the **network interface name**. Also, there are no **`auto`** stanza nor **`allow-hotplug`** stanza to start the network interface `eth0` automatically upon events. Now you are ready to switch the network configuration. Let@@@sq@@@s move your PC to a LAN served by the DHCP. You bring up the **network interface** (the physical interface) `eth0` by assigning the **network configuration** name (the logical interface name) `config1` to it by the following. -------------------- $ sudo ifup eth0=config1 Password: ... -------------------- The interface `eth0` is up, configured by DHCP and connected to LAN. -------------------- $ sudo ifdown eth0=config1 ... -------------------- The interface `eth0` is down and disconnected from LAN. Let@@@sq@@@s move your PC to a LAN served by the static IP. You bring up the **network interface** `eth0` by assigning the **network configuration** name `config2` to it by the following. -------------------- $ sudo ifup eth0=config2 ... -------------------- The interface `eth0` is up, configured with static IP and connected to LAN. The additional parameters given as `dns-\*` configures "`/etc/resolv.conf`" contents. This "`/etc/resolv.conf`" is better manged if the `resolvconf` package is installed. -------------------- $ sudo ifdown eth0=config2 ... -------------------- The interface `eth0` is down and disconnected from LAN, again. Let@@@sq@@@s move your PC to a port on BB-modem connected to the PPPoE served service. You bring up the **network interface** `eth0` by assigning the **network configuration** name `pppoe` to it by the following. -------------------- $ sudo ifup eth0=pppoe ... -------------------- The interface `eth0` is up, configured with PPPoE connection directly to the ISP. -------------------- $ sudo ifdown eth0=pppoe ... -------------------- The interface `eth0` is down and disconnected, again. Let@@@sq@@@s move your PC to a location without LAN or BB-modem but with POTS and modem. You bring up the **network interface** `ppp0` by assigning the **network configuration** name `pots` to it by the following. -------------------- $ sudo ifup ppp0=pots ... -------------------- The interface `ppp0` is up and connected to the Internet with PPP. -------------------- $ sudo ifdown ppp0=pots ... -------------------- The interface `ppp0` is down and disconnected from the Internet. You should check the "`/etc/network/run/ifstate`" file for the current network configuration state of the `ifupdown` system. WARNING: You may need to adjust numbers at the end of `eth\*`, `ppp\*`, etc. if you have multiple network interfaces. ==== Scripting with the ifupdown system The `ifupdown` system automatically runs scripts installed in "`/etc/network/\*/`" while exporting environment variables to scripts. .List of environment variables passed by the ifupdown system [grid="all"] `--------------------`----------------------------------------------------------------------------------------------------------- environment variable value passed --------------------------------------------------------------------------------------------------------------------------------- "`$IFACE`" physical name (interface name) of the interface being processed "`$LOGICAL`" logical name (configuration name) of the interface being processed "`$ADDRFAM`" of the interface "`$METHOD`" of the interface (e.g., "static") "`$MODE`" "start" if run from `ifup`, "stop" if run from `ifdown` "`$PHASE`" as per "`$MODE`", but with finer granularity, distinguishing the `pre-up`, `post-up`, `pre-down` and `post-down` phases "`$VERBOSITY`" indicates whether "`--verbose`" was used; set to 1 if so, 0 if not "`$PATH`" command search path: "`/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`" "`$IF_